Skip to content
Snippets Groups Projects
IntraSearch.cpp 400 KiB
Newer Older
  • Learn to ignore specific revisions
  •       if( !cu.isSepTree() && cu.ispMode )
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #else
          if (!CS::isDualITree(cs) && cu.ispMode)
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
          if( cu.isSepTree() )
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #else
          if (CS::isDualITree(cs))
    #endif
    
          {
            if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
            {
              partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
    
              do
              {
                cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType ).depth = partitioner.currTrDepth;
              } while( partitioner.nextPart( cs ) );
    
              partitioner.exitCurrSplit();
            }
            else
            cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType );
          }
    
          std::vector<TransformUnit*> orgTUs;
    
    
          if( lumaUsesISP )
          {
            CodingUnit& auxCU = saveCS.addCU( cu, partitioner.chType );
            auxCU.ispMode = cu.ispMode;
            saveCS.sps = cu.cs->sps;
            saveCS.addPU( *cu.firstPU, partitioner.chType );
          }
    
    
    
          // create a store for the TUs
          for( const auto &ptu : cs.tus )
          {
            // for split TUs in HEVC, add the TUs without Chroma parts for correct setting of Cbfs
    
            if( lumaUsesISP || pu.contains( *ptu, CHANNEL_TYPE_CHROMA ) )
    
            {
              saveCS.addTU( *ptu, partitioner.chType );
              orgTUs.push_back( ptu );
            }
          }
    
          // SATD pre-selecting.
          int satdModeList[NUM_CHROMA_MODE];
          int64_t satdSortedCost[NUM_CHROMA_MODE];
          for (int i = 0; i < NUM_CHROMA_MODE; i++)
          {
            satdSortedCost[i] = 0; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
            satdModeList[i] = 0;
          }
    
    #if JVET_Z0050_DIMD_CHROMA_FUSION && ENABLE_DIMD
    
    #if JVET_AC0071_DBV
          bool modeIsEnable[NUM_INTRA_MODE + 3]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 3; i++)
          {
            modeIsEnable[i] = 1;
          }
    #else
          bool modeIsEnable[NUM_INTRA_MODE + 2]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 2; i++)
          {
            modeIsEnable[i] = 1;
          }
    #endif
    #else
    #if JVET_AC0071_DBV
    
          bool modeIsEnable[NUM_INTRA_MODE + 2]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 2; i++)
          {
            modeIsEnable[i] = 1;
          }
    #else
    
          bool modeIsEnable[NUM_INTRA_MODE + 1]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 1; i++)
          {
            modeIsEnable[i] = 1;
          }
    
          DistParam distParamSad;
          DistParam distParamSatd;
    
          pu.intraDir[1] = MDLM_L_IDX; // temporary assigned, just to indicate this is a MDLM mode. for luma down-sampling operation.
    
          initIntraPatternChType(cu, pu.Cb());
          initIntraPatternChType(cu, pu.Cr());
          xGetLumaRecPixels(pu, pu.Cb());
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
    
    #if JVET_AA0126_GLM
          if ( PU::isLMCModeEnabled( pu, LM_CHROMA_IDX ) && PU::hasGlmFlag( pu, LM_CHROMA_IDX ) )
          {
    
    #if JVET_AB0092_GLM_WITH_LUMA && JVET_AB0174_CCCM_DIV_FREE
            xGlmSetLumaRefValue(pu, pu.Cb());
    #endif
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
            // Generate all GLM templates at encoder
            xGetLumaRecPixelsGlmAll(pu, pu.Cb());
    
            pu.intraDir[1] = LM_CHROMA_IDX;
            xGetLumaRecPixels(pu, pu.Cb());
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
    
            for ( int mode = LM_CHROMA_IDX; mode <= MMLM_T_IDX; mode++ )
            {
              satdGlmIdcBest[mode - LM_CHROMA_IDX].setAllZero();
              
    
    #if JVET_AB0092_GLM_WITH_LUMA
              CodedCUInfo& relatedCU = ((EncModeCtrlMTnoRQT *)m_modeCtrl)->getBlkInfo(partitioner.currArea());
              if (PU::hasGlmFlag(pu, mode) && !relatedCU.skipGLM)
    #else
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
              if ( PU::hasGlmFlag( pu, mode ) )
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
              {
    
    #if !JVET_AB0092_GLM_WITH_LUMA
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
                for ( int comp = COMPONENT_Cb; comp <= COMPONENT_Cr; comp++ )
                {
                  ComponentID       compID = ComponentID( comp );
    
    Vadim Seregin's avatar
    Vadim Seregin committed
                  ComponentID       compID = COMPONENT_Cb;
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
                  int              idcBest = 0;
                  int64_t         satdBest = 0;
                  GlmIdc&         idcsBest = satdGlmIdcBest[mode - LM_CHROMA_IDX];
                  
                  pu.intraDir[1] = mode;
                  pu.glmIdc.setAllZero();
    
                  xFindBestGlmIdcSATD(pu, compID, idcBest, satdBest );
    
                  idcsBest.setIdc(compID, 0, idcBest);
                  idcsBest.setIdc(compID, 1, idcBest);
    
    
    #if JVET_AB0092_GLM_WITH_LUMA
                  idcsBest.setIdc(COMPONENT_Cr, 0, idcBest);
                  idcsBest.setIdc(COMPONENT_Cr, 1, idcBest);
    #endif
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
                  
                  satdGlmCosts[mode - LM_CHROMA_IDX] += satdBest; // Summing up Cb and Cr cost
    
    #if !JVET_AB0092_GLM_WITH_LUMA
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
                }
                
                if ( !satdGlmIdcBest[0].isActive() )
                {
                  break;
                }
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
              }
            }
          }
    
          pu.glmIdc.setAllZero();
    #endif
    
          
    #if JVET_Z0050_CCLM_SLOPE
          if ( PU::isLMCModeEnabled( pu, LM_CHROMA_IDX ) && PU::hasCclmDeltaFlag( pu, LM_CHROMA_IDX ) )
          {
            // Fill luma reference buffer for the two-sided CCLM
            pu.intraDir[1] = LM_CHROMA_IDX;
            xGetLumaRecPixels(pu, pu.Cb());
    
            for ( int mode = LM_CHROMA_IDX; mode <= MDLM_T_IDX; mode++ )
            {
              satdCclmOffsetsBest[mode - LM_CHROMA_IDX].setAllZero();
              
              if ( PU::hasCclmDeltaFlag( pu, mode ) )
              {
                for ( int comp = COMPONENT_Cb; comp <= COMPONENT_Cr; comp++ )
                {
                  ComponentID       compID = ComponentID( comp );
                  int            deltaBest = 0;
                  int64_t         satdBest = 0;
                  CclmOffsets& offsetsBest = satdCclmOffsetsBest[mode - LM_CHROMA_IDX];
                  
                  pu.intraDir[1] = mode;
                  pu.cclmOffsets.setAllZero();
    
                  xFindBestCclmDeltaSlopeSATD(pu, compID, 0, deltaBest, satdBest );
    
                  offsetsBest.setOffset(compID, 0, deltaBest);
    
    #if MMLM
                  if ( PU::isMultiModeLM( mode ) )
                  {
                    // Set best found values for the first model to get a matching second model
                    pu.cclmOffsets.setOffsets(offsetsBest.cb0, offsetsBest.cr0, 0, 0);
    
                    xFindBestCclmDeltaSlopeSATD(pu, compID, 1, deltaBest, satdBest );
    
                    offsetsBest.setOffset(compID, 1, deltaBest);
                  }
    #endif
    
                  satdCclmCosts[mode - LM_CHROMA_IDX] += satdBest; // Summing up Cb and Cr cost
                }
              }
            }
          }
    
          pu.cclmOffsets.setAllZero();
    #endif
    
    #if JVET_AD0120_LBCCP && MMLM
          std::vector<lmPredFiltModeInfo> miLmPredFiltList;
          miLmPredFiltList.clear();
    #if JVET_AD0188_CCP_MERGE
          CCPModelCandidate ccpCandlmPredFilt[LBCCP_FILTER_MMLMNUM];
    #endif
          PelUnitBuf lmPredFilterStorage[LBCCP_FILTER_MMLMNUM];
          const UnitArea tmpUnitArea(pu.chromaFormat, Area(0, 0, (pu.Cb().width) << getChannelTypeScaleX(CHANNEL_TYPE_CHROMA, pu.chromaFormat), (pu.Cb().height) << getChannelTypeScaleY(CHANNEL_TYPE_CHROMA, pu.chromaFormat)));
          for (uint32_t i = 0; i < LBCCP_FILTER_MMLMNUM; i++)
          {
            lmPredFilterStorage[i] = m_lmPredFiltStorage[i].getBuf(tmpUnitArea);
    #if JVET_AD0188_CCP_MERGE
            ccpCandlmPredFilt[i] = {};
    #endif
          }
          int lmPredFiltIdx = 0;
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if MMLM
          m_encPreRDRun = true;
    #endif
    
          for (int idx = uiMinMode; idx <= uiMaxMode - 1; idx++)
          {
            int mode = chromaCandModes[idx];
            satdModeList[idx] = mode;
            if (PU::isLMCMode(mode) && !PU::isLMCModeEnabled(pu, mode))
            {
              continue;
            }
    
            if ((mode == LM_CHROMA_IDX) || (mode == PLANAR_IDX) || (mode == DM_CHROMA_IDX)
    #if JVET_Z0050_DIMD_CHROMA_FUSION && ENABLE_DIMD
              || (mode == DIMD_CHROMA_IDX)
    
    #endif
    #if JVET_AC0071_DBV
              || (mode == DBV_CHROMA_IDX)
    
    #endif
              ) // only pre-check regular modes and MDLM modes, not including DM, DIMD, Planar, and LM
    
            {
              continue;
            }
            pu.intraDir[1] = mode; // temporary assigned, for SATD checking.
    
    
            int64_t sad = 0;
            int64_t sadCb = 0;
            int64_t satdCb = 0;
            int64_t sadCr = 0;
            int64_t satdCr = 0;
    
            CodingStructure& cs = *(pu.cs);
    
    #if JVET_AD0120_LBCCP && JVET_AD0188_CCP_MERGE
            pu.curCand = {};
    #endif
    
    
            CompArea areaCb = pu.Cb();
            PelBuf orgCb = cs.getOrgBuf(areaCb);
            PelBuf predCb = cs.getPredBuf(areaCb);
    
            m_pcRdCost->setDistParam(distParamSad, orgCb, predCb, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, false);
            m_pcRdCost->setDistParam(distParamSatd, orgCb, predCb, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, true);
            distParamSad.applyWeight = false;
            distParamSatd.applyWeight = false;
    
            if (PU::isLMCMode(mode))
            {
              predIntraChromaLM(COMPONENT_Cb, predCb, pu, areaCb, mode);
            }
            else
            {
    
              initPredIntraParams(pu, pu.Cb(), *pu.cs->sps);
              predIntraAng(COMPONENT_Cb, predCb, pu);
    
            sadCb = distParamSad.distFunc(distParamSad) * 2;
            satdCb = distParamSatd.distFunc(distParamSatd);
            sad += std::min(sadCb, satdCb);
    
            CompArea areaCr = pu.Cr();
            PelBuf orgCr = cs.getOrgBuf(areaCr);
            PelBuf predCr = cs.getPredBuf(areaCr);
    
            m_pcRdCost->setDistParam(distParamSad, orgCr, predCr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, false);
            m_pcRdCost->setDistParam(distParamSatd, orgCr, predCr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, true);
            distParamSad.applyWeight = false;
            distParamSatd.applyWeight = false;
    
            if (PU::isLMCMode(mode))
            {
              predIntraChromaLM(COMPONENT_Cr, predCr, pu, areaCr, mode);
            }
            else
            {
    
              initPredIntraParams(pu, pu.Cr(), *pu.cs->sps);
              predIntraAng(COMPONENT_Cr, predCr, pu);
    
            sadCr = distParamSad.distFunc(distParamSad) * 2;
            satdCr = distParamSatd.distFunc(distParamSatd);
            sad += std::min(sadCr, satdCr);
    
            satdSortedCost[idx] = sad;
    
    #if JVET_AD0120_LBCCP && MMLM
            if(mode == MMLM_CHROMA_IDX)
            {
              lmPredFilterStorage[lmPredFiltIdx].Cb().copyFrom(predCb);
              lmPredFilterStorage[lmPredFiltIdx].Cr().copyFrom(predCr);
    #if JVET_AD0188_CCP_MERGE
              ccpCandlmPredFilt[lmPredFiltIdx] = pu.curCand;
    #endif
            }
    #endif
    
    
    #if JVET_AB0143_CCCM_TS
    #if MMLM
    
    #if JVET_AC0054_GLCCCM
          uint32_t chromaCandCccmModes[CCCM_NUM_MODES] = { LM_CHROMA_IDX, MDLM_L_IDX, MDLM_T_IDX, MMLM_CHROMA_IDX, MMLM_L_IDX, MMLM_T_IDX
                                                         , LM_CHROMA_IDX, MDLM_L_IDX, MDLM_T_IDX, MMLM_CHROMA_IDX, MMLM_L_IDX, MMLM_T_IDX };
    #else
    
    Vadim Seregin's avatar
    Vadim Seregin committed
          uint32_t chromaCandCccmModes[CCCM_NUM_MODES] = { LM_CHROMA_IDX, MDLM_L_IDX, MDLM_T_IDX, MMLM_CHROMA_IDX, MMLM_L_IDX, MMLM_T_IDX };
    
    #endif
    #else
    #if JVET_AC0054_GLCCCM
          uint32_t chromaCandCccmModes[CCCM_NUM_MODES] = { LM_CHROMA_IDX, MDLM_L_IDX, MDLM_T_IDX
                                                         , LM_CHROMA_IDX, MDLM_L_IDX, MDLM_T_IDX };
    
    Vadim Seregin's avatar
    Vadim Seregin committed
          uint32_t chromaCandCccmModes[CCCM_NUM_MODES] = { LM_CHROMA_IDX, MDLM_L_IDX, MDLM_T_IDX };
    
    #if JVET_AD0202_CCCM_MDF   
          int satdCccmFilterIndex[TOTAL_NUM_CCCM_MODES];
    #if JVET_AC0054_GLCCCM
          int satdCccmFlagList[TOTAL_NUM_CCCM_MODES];
    #endif
    #else
    
    #if JVET_AC0054_GLCCCM
          int satdCccmFlagList[CCCM_NUM_MODES];
    #endif
    
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING
    
    #if JVET_AD0202_CCCM_MDF  
          int64_t satdCccmSortedCost[2][TOTAL_NUM_CCCM_MODES];
          int satdCccmModeList[2][TOTAL_NUM_CCCM_MODES];
          for (int i = 0; i < CCCM_NUM_PRED_FILTER; i++)
          {
            int startIdx = i * CCCM_NUM_MODES;
            for (int j = 0; j < CCCM_NUM_MODES; j++)
            {
              int currCccmModeIdx = startIdx + j;
              satdCccmSortedCost[0][currCccmModeIdx] = LLONG_MAX; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
              satdCccmSortedCost[1][currCccmModeIdx] = LLONG_MAX; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
              satdCccmModeList[0][currCccmModeIdx] = chromaCandCccmModes[j];
              satdCccmModeList[1][currCccmModeIdx] = chromaCandCccmModes[j];
    #if JVET_AC0054_GLCCCM
              satdCccmFlagList[currCccmModeIdx] = j < (CCCM_NUM_MODES / 2) ? 1 : 2; // 1: cccm, 2: glCccm
    #endif
              satdCccmFilterIndex[currCccmModeIdx] = i;
            }
          }
    #else
    
          int64_t satdCccmSortedCost[2][CCCM_NUM_MODES];
          int satdCccmModeList[2][CCCM_NUM_MODES];
    
          for (int i = 0; i < CCCM_NUM_MODES; i++)
          {
            satdCccmSortedCost[0][i] = LLONG_MAX; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
            satdCccmSortedCost[1][i] = LLONG_MAX; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
            satdCccmModeList[0][i] = chromaCandCccmModes[i];
            satdCccmModeList[1][i] = chromaCandCccmModes[i];
    
    #if JVET_AC0054_GLCCCM
            satdCccmFlagList[i] = i < (CCCM_NUM_MODES / 2) ? 1 : 2; // 1: cccm, 2: glCccm
    #endif
    
          int64_t bestCccmCost[2] = { LLONG_MAX, LLONG_MAX};
    
          bool isCccmFullEnabled = PU::cccmSingleModeAvail(pu, LM_CHROMA_IDX);
          bool isCccmLeftEnabled = PU::isLeftCccmMode(pu, MDLM_L_IDX);
          bool isCccmTopEnabled = PU::isTopCccmMode(pu, MDLM_T_IDX);
    #if MMLM
          bool isMultiCccmFullEnabled = PU::cccmMultiModeAvail(pu, MMLM_CHROMA_IDX);
          bool isMultiCccmLeftEnabled = PU::cccmMultiModeAvail(pu, MMLM_L_IDX);
          bool isMultiCccmTopEnabled = PU::cccmMultiModeAvail(pu, MMLM_T_IDX);
    #endif
    
    #if JVET_AD0202_CCCM_MDF
          bool isMultiCccmFullEnabled2 = PU::isMultiCccmWithMdf(pu, MMLM_CHROMA_IDX);
          bool isMultiCccmLeftEnabled2 = PU::isMultiCccmWithMdf(pu, MMLM_L_IDX);
          bool isMultiCccmTopEnabled2 = PU::isMultiCccmWithMdf(pu, MMLM_T_IDX);
    #endif
    
    
          const UnitArea localUnitArea(cs.area.chromaFormat, Area(0, 0, (pu.Cb().width) << 1, (pu.Cb().height) << 1));
    
    #if JVET_AD0202_CCCM_MDF
          PelUnitBuf cccmStorage[2][TOTAL_NUM_CCCM_MODES];
    #else
    
          PelUnitBuf cccmStorage[2][CCCM_NUM_MODES];
    
    #if JVET_AD0202_CCCM_MDF
          pu.cccmNoSubFlag = 1;
          xGetLumaRecPixels(pu, pu.Cb());
          pu.cccmNoSubFlag = 0;
          xGetLumaRecPixels(pu, pu.Cb());
          xGetLumaRecPixels(pu, pu.Cb(), 1);
          xGetLumaRecPixels(pu, pu.Cb(), 2);
          xGetLumaRecPixels(pu, pu.Cb(), 3);
    #endif
    
    
          for (int sub = 0; sub < pu.cu->slice->getSPS()->getUseCccm(); sub++)
          {
            pu.cccmNoSubFlag = sub;
    
    
            xGetLumaRecPixels(pu, pu.Cb());
    
    
            bool isCCCMEnabled = false;
    
            for (int idx = 0; idx < CCCM_NUM_MODES; idx++)
            {
    
    #if JVET_AC0054_GLCCCM
              if (sub && idx >= CCCM_NUM_MODES / 2)
              {
                continue;
              }
    #endif
    
              int mode = chromaCandCccmModes[idx];
              if (idx == 0)
              {
                isCCCMEnabled = isCccmFullEnabled;
                pu.cccmFlag = 1;
              }
              else if (idx == 1)
              {
                isCCCMEnabled = isCccmLeftEnabled;
                pu.cccmFlag = 2;
              }
              else if (idx == 2)
              {
                isCCCMEnabled = isCccmTopEnabled;
                pu.cccmFlag = 3;
              }
    #if MMLM
              else if (idx == 3)
              {
                isCCCMEnabled = isMultiCccmFullEnabled;
                pu.cccmFlag = 1;
              }
              else if (idx == 4)
              {
                isCCCMEnabled = isMultiCccmLeftEnabled;
                pu.cccmFlag = 2;
              }
              else if (idx == 5)
              {
                isCCCMEnabled = isMultiCccmTopEnabled;
                pu.cccmFlag = 3;
              }
    #endif
    
                
    #if JVET_AC0054_GLCCCM
              if (idx < CCCM_NUM_MODES / 2)
              {
                pu.glCccmFlag = 0;
              }
              else
              {
                pu.glCccmFlag = 1;
    #if MMLM
                isCCCMEnabled = idx == 6 ? isCccmFullEnabled
                : idx == 7 ? isCccmLeftEnabled
                : idx == 8 ? isCccmTopEnabled
                : idx == 9 ? isMultiCccmFullEnabled
                : idx == 10 ? isMultiCccmLeftEnabled : isMultiCccmTopEnabled;
                pu.cccmFlag =  idx == 6 ? 1
                : idx == 7 ? 2
                : idx == 8 ? 3
                : idx == 9 ? 1
                : idx == 10 ? 2 : 3;
    #else
                isCCCMEnabled = idx == 3 ? isCccmFullEnabled
                : idx == 4 ? isCccmLeftEnabled : isCccmTopEnabled;
                pu.cccmFlag =   idx == 3 ? 1
                : idx == 4 ? 2 : 3;
    #endif
    
    
    #if JVET_AD0202_CCCM_MDF
                if (isCCCMEnabled)
                {
                  if (m_skipCCCMwithMdfSATD)
                  {
                    if (m_isCccmWithMdfEnabledInRdo[4][mode] == 0)
                    {
                      continue;
                    }
                  }
                }
    #endif
    
    #if JVET_AD0202_CCCM_MDF
                for (int32_t filterIdx = 0; filterIdx < CCCM_NUM_PRED_FILTER; filterIdx++)
                {
                if (filterIdx > 0)
                {
                  if (sub == 1 || idx > 5)
                  {
                    continue;
                  }
    
                  if (m_skipCCCMwithMdfSATD)
                  {
                    if (m_isCccmWithMdfEnabledInRdo[filterIdx][mode] == 0)
                    {
                      continue;
                    }
                  }
    
                  if (mode == MMLM_CHROMA_IDX && !isMultiCccmFullEnabled2)
                  {
                    continue;
                  }
                  else if (mode == MMLM_L_IDX && !isMultiCccmLeftEnabled2)
                  {
                    continue;
                  }
                  else if (mode == MMLM_T_IDX && !isMultiCccmTopEnabled2)
                  {
                    continue;
                  }
                }
                else if (sub == 0 && idx <= 5)
                {
                  if (m_skipCCCMwithMdfSATD)
                  {
                    if (m_isCccmWithMdfEnabledInRdo[filterIdx][mode] == 0)
                    {
                      continue;
                    }
                  }
                }
    
                pu.cccmMultiFilterIdx = filterIdx;
    #endif
    
                pu.intraDir[1] = mode; // temporary assigned, for SATD checking.
    
                if ( ( sub == 1 ) && m_skipCCCMSATD )
                {
                  if (m_isCccmNoSubModeEnabledInRdo[mode] == 0)
                  {
                    continue;
                  }
                }
    
                int64_t sad = 0;
                int64_t sadCb = 0;
                int64_t satdCb = 0;
                int64_t sadCr = 0;
                int64_t satdCr = 0;
                CodingStructure& cs = *(pu.cs);
    
                DistParam distParamSadCb;
                DistParam distParamSatdCb;
                DistParam distParamSadCr;
                DistParam distParamSatdCr;
    
    
    #if JVET_AD0202_CCCM_MDF
                const int cccmBufferIdx = filterIdx * CCCM_NUM_MODES + idx;
                cccmStorage[sub][cccmBufferIdx] = m_cccmStorage[sub][cccmBufferIdx].getBuf(localUnitArea);
    #else
    
                cccmStorage[sub][idx] = m_cccmStorage[sub][idx].getBuf(localUnitArea);
    
    
                CompArea areaCb = pu.Cb();
                PelBuf orgCb = cs.getOrgBuf(areaCb);
                CompArea areaCr = pu.Cr();
                PelBuf orgCr = cs.getOrgBuf(areaCr);
    
    
    #if JVET_AD0202_CCCM_MDF
                m_pcRdCost->setDistParam(distParamSadCb, orgCb, cccmStorage[sub][cccmBufferIdx].Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, false);
                m_pcRdCost->setDistParam(distParamSatdCb, orgCb, cccmStorage[sub][cccmBufferIdx].Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, true);
    #else
    
                m_pcRdCost->setDistParam(distParamSadCb, orgCb, cccmStorage[sub][idx].Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, false);
                m_pcRdCost->setDistParam(distParamSatdCb, orgCb, cccmStorage[sub][idx].Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, true);
    
                distParamSadCb.applyWeight = false;
                distParamSatdCb.applyWeight = false;
    
    #if JVET_AD0202_CCCM_MDF
                m_pcRdCost->setDistParam(distParamSadCr, orgCr, cccmStorage[sub][cccmBufferIdx].Cr(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, false);
                m_pcRdCost->setDistParam(distParamSatdCr, orgCr, cccmStorage[sub][cccmBufferIdx].Cr(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, true);
    #else
    
                m_pcRdCost->setDistParam(distParamSadCr, orgCr, cccmStorage[sub][idx].Cr(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, false);
                m_pcRdCost->setDistParam(distParamSatdCr, orgCr, cccmStorage[sub][idx].Cr(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, true);
    
                distParamSadCr.applyWeight = false;
                distParamSatdCr.applyWeight = false;
    
    
    #if JVET_AD0202_CCCM_MDF
                predIntraCCCM(pu, cccmStorage[sub][cccmBufferIdx].Cb(), cccmStorage[sub][cccmBufferIdx].Cr(), mode);
    #else
    
                predIntraCCCM(pu, cccmStorage[sub][idx].Cb(), cccmStorage[sub][idx].Cr(), mode);
    
    #if JVET_AD0188_CCP_MERGE
    #if JVET_AD0202_CCCM_MDF
                m_ccmParamsStorage[sub][cccmBufferIdx] = pu.curCand;
    #else
                m_ccmParamsStorage[sub][idx] = pu.curCand;
    #endif
    #endif
    
                sadCb = distParamSadCb.distFunc(distParamSadCb) * 2;
                satdCb = distParamSatdCb.distFunc(distParamSatdCb);
                sad += std::min(sadCb, satdCb);
                sadCr = distParamSadCr.distFunc(distParamSadCr) * 2;
                satdCr = distParamSatdCr.distFunc(distParamSatdCr);
                sad += std::min(sadCr, satdCr);
    
    
    #if JVET_AD0202_CCCM_MDF
                satdCccmSortedCost[sub][cccmBufferIdx] = sad;
    #else
    
                satdCccmSortedCost[sub][idx] = sad;
    
    
                if (sad < bestCccmCost[sub])
                {
                  bestCccmCost[sub] = sad;
                }
    
    #if JVET_AC0054_GLCCCM
          int tempGlCccmFlag = 0;
    #endif
    
          int tempCccmIdx = 0;
          int64_t tempCccmCost = 0;
    
    #if JVET_AD0202_CCCM_MDF 
          for (int i = 1; i < 7; i++)
    #else
    
          for (int i = 1; i < CCCM_NUM_MODES; i++)
    
    #if MMLM
          for (int i = 1; i < 4; i++)
    #else
          for (int i = 1; i < 3; i++)
    
    #if JVET_AD0202_CCCM_MDF
            for (int j = i + 1; j < VALID_NUM_CCCM_MODES; j++)
    #else
    
            for (int j = i + 1; j < CCCM_NUM_MODES; j++)
    
            {
              if (satdCccmSortedCost[0][j] < satdCccmSortedCost[0][i])
              {
                tempCccmIdx = satdCccmModeList[0][i];
                satdCccmModeList[0][i] = satdCccmModeList[0][j];
                satdCccmModeList[0][j] = tempCccmIdx;
    
                tempCccmCost = satdCccmSortedCost[0][i];
                satdCccmSortedCost[0][i] = satdCccmSortedCost[0][j];
                satdCccmSortedCost[0][j] = tempCccmCost;
    
    #if JVET_AC0054_GLCCCM
                tempGlCccmFlag = satdCccmFlagList[i];
                satdCccmFlagList[i] = satdCccmFlagList[j];
                satdCccmFlagList[j] = tempGlCccmFlag;
    
    #endif
    #if JVET_AD0202_CCCM_MDF 
                tempCccmIdx = satdCccmFilterIndex[i];
                satdCccmFilterIndex[i] = satdCccmFilterIndex[j];
                satdCccmFilterIndex[j] = tempCccmIdx;
    
              }
            }
          }
    
    #if MMLM
          bool isCccmModeEnabledInRdo[2][MMLM_T_IDX + 1] = { false };
    
          isCccmModeEnabledInRdo[0][satdCccmModeList[0][0]] = true;
    
    #if JVET_AC0054_GLCCCM
          bool isGlCccmModeEnabledInRdo[MMLM_T_IDX + 1] = { false };
    
          if (satdCccmFlagList[0] == 2)
          {
            isCccmModeEnabledInRdo[0][satdCccmModeList[0][0]] = false;
            isGlCccmModeEnabledInRdo[satdCccmModeList[0][0]] = true;
          }
    #endif
    
    #endif
    #if JVET_AD0202_CCCM_MDF 
          bool isCccmWithMulDownSamplingEnabledInRdo[MMLM_T_IDX + 1][CCCM_NUM_PRED_FILTER] = { false };
          isCccmWithMulDownSamplingEnabledInRdo[satdCccmModeList[0][0]][satdCccmFilterIndex[0]] = true;
          for (int i = 1; i < 7; i++)
    #else
    
          for (int i = 1; i < 4; i++)
    
    #else
          bool isCccmModeEnabledInRdo[MDLM_T_IDX + 1] = { false };
          isCccmModeEnabledInRdo[satdCccmModeList[0]] = true;
    
    #if JVET_AC0054_GLCCCM
          bool isGlCccmModeEnabledInRdo[MDLM_T_IDX + 1] = { false };
          if (satdCccmFlagList[0] == 2)
          {
            isCccmModeEnabledInRdo[0][satdCccmModeList[0][0]] = false;
            isGlCccmModeEnabledInRdo[satdCccmModeList[0][0]] = true;
          }
    #endif
    
          for (int i = 1; i < 3; i++)
    #endif
          {
            if (satdCccmSortedCost[0][i] >= 1.15 * bestCccmCost[0])
            {
              break;
            }
    
    #if JVET_AC0054_GLCCCM
            if (satdCccmFlagList[i] == 2)
            {
              isGlCccmModeEnabledInRdo[satdCccmModeList[0][i]] = true;
            }
            else
            {
    
    #if JVET_AD0202_CCCM_MDF 
              isCccmWithMulDownSamplingEnabledInRdo[satdCccmModeList[0][i]][satdCccmFilterIndex[i]] = true;
    #else
    
              isCccmModeEnabledInRdo[0][satdCccmModeList[0][i]] = true;
    
            isCccmModeEnabledInRdo[0][satdCccmModeList[0][i]] = true;
    
    #if JVET_AD0202_CCCM_MDF
          if (m_skipCCCMwithMdfSATD == false)
          {
            m_skipCCCMwithMdfSATD = true;
            for (int i = LM_CHROMA_IDX; i <= MMLM_T_IDX; i++)
            {
              m_isCccmWithMdfEnabledInRdo[4][i] = isGlCccmModeEnabledInRdo[i] ? 1 : 0;
    
              for (int j = 0; j < 4; j++)
              {
                m_isCccmWithMdfEnabledInRdo[j][i] = isCccmWithMulDownSamplingEnabledInRdo[i][j] ? 1 : 0;
              }
            }
          }
    #endif
    
    
          if (pu.cu->slice->getSPS()->getUseCccm() == 2)
          {
            if (bestCccmCost[1] < bestCccmCost[0])
            {
    #if MMLM
    
    #if JVET_AD0202_CCCM_MDF
              for (int i = 1; i < 7; i++)
    #else
    
              for (int i = 0; i < 4; i++)
    
    #else
              for (int i = 0; i < 3; i++)
    #endif
              {
    
    #if JVET_AD0202_CCCM_MDF
                if (isCccmWithMulDownSamplingEnabledInRdo[satdCccmModeList[0][i]][satdCccmFilterIndex[i]] && (satdCccmSortedCost[0][i] >= 1.2 * bestCccmCost[1]))
                {
                  isCccmWithMulDownSamplingEnabledInRdo[satdCccmModeList[0][i]][satdCccmFilterIndex[i]] = false;
                }
    #else
    
                if (isCccmModeEnabledInRdo[0][satdCccmModeList[0][i]] && (satdCccmSortedCost[0][i] >= 1.2 * bestCccmCost[1]))
                {
                  isCccmModeEnabledInRdo[0][satdCccmModeList[0][i]] = false;
                }
    
                else
                {
                  bestCccmCost[0] = satdCccmSortedCost[0][i];
                }
              }
            }
            else
            {
              bestCccmCost[1] = bestCccmCost[0];
            }
    
            tempCccmIdx = 0;
            tempCccmCost = 0;
            for (int i = 1; i < 2; i++)
            {
              for (int j = i + 1; j < 6; j++)
              {
                if (satdCccmSortedCost[1][j] < satdCccmSortedCost[1][i])
                {
                  tempCccmIdx = satdCccmModeList[1][i];
                  satdCccmModeList[1][i] = satdCccmModeList[1][j];
                  satdCccmModeList[1][j] = tempCccmIdx;
    
                  tempCccmCost = satdCccmSortedCost[1][i];
                  satdCccmSortedCost[1][i] = satdCccmSortedCost[1][j];
                  satdCccmSortedCost[1][j] = tempCccmCost;
                }
              }
            }
    
            isCccmModeEnabledInRdo[1][satdCccmModeList[1][0]] = true;
            for (int i = 1; i < 2; i++)
            {
              if (satdCccmSortedCost[1][i] >= CCCM_NO_SUB_WEIGHT * bestCccmCost[1])
              {
    
    #if JVET_AC0054_GLCCCM
                if (satdCccmSortedCost[1][i - 1] > bestCccmCost[0] && satdCccmFlagList[0] != 2)
    #else
    
                if (satdCccmSortedCost[1][i - 1] > bestCccmCost[0])
    
                {
                  bestCccmCost[0] = satdCccmSortedCost[1][i - 1];
                }
                bestCccmCost[1] = satdCccmSortedCost[1][i - 1];
    
                break;
              }
              isCccmModeEnabledInRdo[1][satdCccmModeList[1][i]] = true;
            }
    
    #if JVET_AD0202_CCCM_MDF
            bestCccmCost[1] = (isCccmModeEnabledInRdo[1][satdCccmModeList[1][1]] && satdCccmSortedCost[1][0] < satdCccmSortedCost[1][1]) ? satdCccmSortedCost[1][1] : satdCccmSortedCost[1][0];
            bestCccmCost[0] = (bestCccmCost[1] > bestCccmCost[0]) ? bestCccmCost[1] : bestCccmCost[0];
            bestCccmCost[0] = (satdCccmSortedCost[0][0] > bestCccmCost[0]) ? satdCccmSortedCost[0][0] : bestCccmCost[0];
    #endif
    
            if (m_skipCCCMSATD == false)
            {
              m_skipCCCMSATD = true;
              for (int i = 0; i < (MMLM_T_IDX + 1); i++)
              {
                m_isCccmNoSubModeEnabledInRdo[i] = isCccmModeEnabledInRdo[1][i];
              }
            }
          }
    
          pu.cccmFlag = 0;
          pu.cccmNoSubFlag = 0;
    
    #if JVET_AC0054_GLCCCM
          pu.glCccmFlag = 0;
    #endif
    
    #if JVET_AD0202_CCCM_MDF 
          pu.cccmMultiFilterIdx = 0;
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
          int64_t satdCccmSortedCost[CCCM_NUM_MODES];
          int satdCccmModeList[CCCM_NUM_MODES];
    
          for (int i = 0; i < CCCM_NUM_MODES; i++)
    
          {
            satdCccmSortedCost[i] = LLONG_MAX; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
            satdCccmModeList[i] = chromaCandCccmModes[i];
          }
          int64_t bestCccmCost = LLONG_MAX;
    
          bool isCccmFullEnabled = PU::cccmSingleModeAvail(pu, LM_CHROMA_IDX);
          bool isCccmLeftEnabled = PU::isLeftCccmMode(pu, MDLM_L_IDX);
          bool isCccmTopEnabled = PU::isTopCccmMode(pu, MDLM_T_IDX);
    #if MMLM
          bool isMultiCccmFullEnabled = PU::cccmMultiModeAvail(pu, MMLM_CHROMA_IDX);
          bool isMultiCccmLeftEnabled = PU::cccmMultiModeAvail(pu, MMLM_L_IDX);
          bool isMultiCccmTopEnabled = PU::cccmMultiModeAvail(pu, MMLM_T_IDX);
    #endif
    
          const UnitArea localUnitArea(cs.area.chromaFormat, Area(0, 0, (pu.Cb().width) << 1, (pu.Cb().height) << 1));
    
    Vadim Seregin's avatar
    Vadim Seregin committed
          PelUnitBuf cccmStorage[CCCM_NUM_MODES];
    
    
          pu.cccmFlag = 1;
          xGetLumaRecPixels(pu, pu.Cb());
    
    
    Vadim Seregin's avatar
    Vadim Seregin committed
          bool isCCCMEnabled = false;
    
          for (int idx = 0; idx < CCCM_NUM_MODES; idx++)
    
          {
            int mode = chromaCandCccmModes[idx];
            if (idx == 0)
            {
              isCCCMEnabled = isCccmFullEnabled;
              pu.cccmFlag = 1;
            }
            else if (idx == 1)
            {
              isCCCMEnabled = isCccmLeftEnabled;
              pu.cccmFlag = 2;
            }
            else if (idx == 2)
            {
              isCCCMEnabled = isCccmTopEnabled;
              pu.cccmFlag = 3;
            }
    #if MMLM
            else if (idx == 3)
            {
              isCCCMEnabled = isMultiCccmFullEnabled;
              pu.cccmFlag = 1;
            }
            else if (idx == 4)
            {
              isCCCMEnabled = isMultiCccmLeftEnabled;
              pu.cccmFlag = 2;
            }
            else if (idx == 5)
            {
              isCCCMEnabled = isMultiCccmTopEnabled;
              pu.cccmFlag = 3;
            }
    #endif
    
            if (isCCCMEnabled)
            {
              pu.intraDir[1] = mode; // temporary assigned, for SATD checking.
    
              int64_t sad = 0;
              int64_t sadCb = 0;
              int64_t satdCb = 0;
              int64_t sadCr = 0;
              int64_t satdCr = 0;
              CodingStructure& cs = *(pu.cs);
    
              DistParam distParamSadCb;
              DistParam distParamSatdCb;
              DistParam distParamSadCr;
              DistParam distParamSatdCr;
    
              cccmStorage[idx] = m_cccmStorage[idx].getBuf(localUnitArea);
    
              CompArea areaCb = pu.Cb();
              PelBuf orgCb = cs.getOrgBuf(areaCb);
              CompArea areaCr = pu.Cr();
              PelBuf orgCr = cs.getOrgBuf(areaCr);
    
              m_pcRdCost->setDistParam(distParamSadCb, orgCb, cccmStorage[idx].Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, false);
              m_pcRdCost->setDistParam(distParamSatdCb, orgCb, cccmStorage[idx].Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, true);
              distParamSadCb.applyWeight = false;
              distParamSatdCb.applyWeight = false;
              m_pcRdCost->setDistParam(distParamSadCr, orgCr, cccmStorage[idx].Cr(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, false);
              m_pcRdCost->setDistParam(distParamSatdCr, orgCr, cccmStorage[idx].Cr(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, true);
    
              distParamSadCr.applyWeight = false;
              distParamSatdCr.applyWeight = false;
    
    
              predIntraCCCM(pu, cccmStorage[idx].Cb(), cccmStorage[idx].Cr(), mode);
    
    #if JVET_AD0188_CCP_MERGE
              m_ccmParamsStorage[idx] = pu.curCand;
    #endif
    
    
              sadCb = distParamSadCb.distFunc(distParamSadCb) * 2;
              satdCb = distParamSatdCb.distFunc(distParamSatdCb);
              sad += std::min(sadCb, satdCb);
              sadCr = distParamSadCr.distFunc(distParamSadCr) * 2;
              satdCr = distParamSatdCr.distFunc(distParamSatdCr);
              sad += std::min(sadCr, satdCr);
    
              satdCccmSortedCost[idx] = sad;
    
              if (sad < bestCccmCost)
              {
                bestCccmCost = sad;
              }
            }
          }
    
          int tempCccmIdx = 0;
          int64_t tempCccmCost = 0;
    #if MMLM
          for (int i = 1; i < 4; i++)
    #else
          for (int i = 1; i < 3; i++)
    #endif
          {
    
    Vadim Seregin's avatar
    Vadim Seregin committed
            for (int j = i + 1; j < CCCM_NUM_MODES; j++)
    
            {
              if (satdCccmSortedCost[j] < satdCccmSortedCost[i])
              {
                tempCccmIdx = satdCccmModeList[i];
                satdCccmModeList[i] = satdCccmModeList[j];
                satdCccmModeList[j] = tempCccmIdx;
    
                tempCccmCost = satdCccmSortedCost[i];
                satdCccmSortedCost[i] = satdCccmSortedCost[j];
                satdCccmSortedCost[j] = tempCccmCost;
              }
            }