Skip to content
Snippets Groups Projects
IntraSearch.cpp 565 KiB
Newer Older
  • Learn to ignore specific revisions
  •         }
            if( !cu.ispMode && !cu.bdpcmMode && csBest->cost < bestCostNonBDPCM )
    
              bestCostNonBDPCM = csBest->cost;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
          if( m_pcEncCfg->getFastLocalDualTreeMode() )
    
            if( cu.isConsIntra() && !cu.slice->isIntra() && csBest->cost != MAX_DOUBLE && costInterCU != COST_UNKNOWN && mode >= 0 )
    
              if( m_pcEncCfg->getFastLocalDualTreeMode() == 2 )
    
                //Note: only try one intra mode, which is especially useful to reduce EncT for LDB case (around 4%)
    
          if (sps.getUseColorTrans() && !CS::isDualITree(cs))
          {
            if ((m_pcEncCfg->getRGBFormatFlag() && !cu.colorTransform) && csBest->cost != MAX_DOUBLE && bestCS->cost != MAX_DOUBLE && mode >= 0)
            {
              if (csBest->cost > bestCS->cost)
              {
                break;
              }
            }
          }
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
    #if INTRA_TRANS_ENC_OPT
        if (setSkipTimdControl && regAngCost != MAX_DOUBLE && timdAngCost != MAX_DOUBLE)
        {
          if (regAngCost * 1.3 < timdAngCost)
          {
            m_skipTimdLfnstMtsPass = true;
          }
        }
    
    #if JVET_AJ0112_REGRESSION_SGPM
        if (setSkipSgpmControl)
        {
          if (regAngCost != MAX_DOUBLE && sgpmCost != MAX_DOUBLE && regAngCost * 1.5 < sgpmCost)
          {
            m_skipSgpmLfnstMtsPass = true;
          }
        }
    #endif
    
    #if JVET_AJ0061_TIMD_MERGE
        for (int i = 0; i < NUM_TIMD_MERGE_MODES && setSkipTimdMrgControl; i++)
        {
          if (timdMrgAngCost[i] != MAX_DOUBLE && (regAngCost * 1.3 < timdMrgAngCost[i] || timdAngCost * 1.3 < timdMrgAngCost[i]))
          {
            m_skipTimdMrgLfnstMtsPass = true;
          }
          if (timdMrgAngCost[i] != MAX_DOUBLE && timdAngCost != MAX_DOUBLE && timdMrgAngCost[i] * 2 < timdAngCost)
          {
            m_skipTimdLfnstMtsPass = true;
          }
        }
    #endif
    
    #if JVET_AH0076_OBIC
        if (setSkipDimdControl)
        {
          if (regAngCost != MAX_DOUBLE && dimdAngCost != MAX_DOUBLE && regAngCost * 1.5 < dimdAngCost)
          {
            m_skipDimdLfnstMtsPass = true;
          }
          if (regAngCost != MAX_DOUBLE && obicAngCost != MAX_DOUBLE && regAngCost * 1.5 < obicAngCost)
          {
            m_skipObicLfnstMtsPass = true;
          }
        }
    
    #endif
    #if JVET_AJ0082_MM_EIP
        if (setSkipEipControl)
        {
          if(regAngCost != MAX_DOUBLE && eipCost != MAX_DOUBLE && regAngCost * EIP_LNFST_MTS_RATE < eipCost)
          {
            m_skipEipLfnstMtsPass = true;
          }
        }
    
    #endif
    #if JVET_AJ0249_NEURAL_NETWORK_BASED
        if (setSkipNnControl && regAngCostSupp != MAX_DOUBLE && nnAngCost != MAX_DOUBLE && regAngCostSupp * 1.5 < nnAngCost)
        {
          m_skipNnLfnstMtsPass = true;
        }
    
        cu.ispMode = uiBestPUMode.ispMod;
    
        cu.lfnstIdx = bestLfnstIdx;
    
            cs.useSubStructure(*csBest, partitioner.chType, pu, true, true, KEEP_PRED_AND_RESI_SIGNALS, KEEP_PRED_AND_RESI_SIGNALS, true);
    
            cs.useSubStructure(*csBest, partitioner.chType, pu.singleChan(CHANNEL_TYPE_LUMA), true, true, KEEP_PRED_AND_RESI_SIGNALS,
    
    #if JVET_AB0061_ITMP_BV_FOR_IBC
        if (uiBestPUMode.tmpFlag)
        {
          pu.interDir               = 1;             // use list 0 for IBC mode
          pu.refIdx[REF_PIC_LIST_0] = MAX_NUM_REF;   // last idx in the list
          pu.mv[0]                  = csBest->pus[0]->mv[0];
          pu.bv                     = csBest->pus[0]->bv;
        }
    #endif
    
        if( validReturn )
        {
          //=== update PU data ====
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if JVET_V0130_INTRA_TMP
          cu.tmpFlag = uiBestPUMode.tmpFlag;
    
    #endif
    #if JVET_AD0086_ENHANCED_INTRA_TMP
          cu.tmpIdx        = uiBestPUMode.tmpIdx;
          cu.tmpFusionFlag = uiBestPUMode.tmpFusionFlag;
          cu.tmpFlmFlag    = uiBestPUMode.tmpFlmFlag;
    
    #if JVET_AG0136_INTRA_TMP_LIC
          cu.tmpLicFlag = uiBestPUMode.tmpLicFlag;
          cu.ibcLicFlag = uiBestPUMode.tmpLicFlag;
          cu.ibcLicIdx = uiBestPUMode.tmpLicIdc;
    #endif
    
          cu.tmpIsSubPel   = uiBestPUMode.tmpIsSubPel;
          cu.tmpSubPelIdx  = uiBestPUMode.tmpSubPelIdx;
    
    #if JVET_AH0200_INTRA_TMP_BV_REORDER
          cu.tmpFracIdx    = uiBestPUMode.tmpFracIdx;
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
          cu.mipFlag = uiBestPUMode.mipFlg;
    
          pu.mipTransposedFlag             = uiBestPUMode.mipTrFlg;
    
          pu.multiRefIdx = uiBestPUMode.mRefId;
          pu.intraDir[ CHANNEL_TYPE_LUMA ] = uiBestPUMode.modeId;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if ENABLE_DIMD
          cu.dimd = bestDimdMode;
          if (cu.dimd)
          {
            CHECK(pu.multiRefIdx > 0, "use of DIMD");
          }
    
    #endif
    #if JVET_AH0076_OBIC
          cu.obicFlag = bestObicMode;
          cu.mipDimdMode = bestMipDimd;
          if (cu.obicFlag)
          {
            pu.intraDir[ CHANNEL_TYPE_LUMA ] = cu.obicMode[0];
            cu.dimd = true;
          }
    
    #endif
    #if JVET_AJ0112_REGRESSION_SGPM
          cu.sgpmDimdMode = bestSgpmDimd;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
          cu.bdpcmMode = bestBDPCMMode;
    
    Keming Cao's avatar
    Keming Cao committed
    #if JVET_W0123_TIMD_FUSION
          cu.timd = bestTimdMode;
    
    #if JVET_AJ0146_TIMDSAD
          cu.timdSad = bestTimdModeSad;
    #endif
    
    Keming Cao's avatar
    Keming Cao committed
          if (cu.timd)
          {
    
    #if JVET_AJ0146_TIMDSAD
            if (cu.timdSad)
            {
              CHECK(uiBestPUMode.modeId != cu.timdModeSad, "mode id should match timdModeSad");
            }
    #if !JVET_AJ0061_TIMD_MERGE			  
            else
            {
              CHECK(uiBestPUMode.modeId != cu.timdMode, "mode id should match timdMode");
            }
    #endif
            pu.intraDir[ CHANNEL_TYPE_LUMA ] = cu.timdSad ? cu.timdModeSad : cu.timdMode;
    #else
    
    Keming Cao's avatar
    Keming Cao committed
            pu.intraDir[ CHANNEL_TYPE_LUMA ] = cu.timdMode;
    
    #if JVET_AJ0061_TIMD_MERGE
            if (!cu.timdMrg)
            {
              cu.timdmTrType[NUM_TIMD_MERGE_MODES][0] = bestTimdTrType[0];
              cu.timdmTrType[NUM_TIMD_MERGE_MODES][1] = bestTimdTrType[1];
            }
    #endif
          }
    #if JVET_AJ0061_TIMD_MERGE
          cu.timdMrg = bestTimdMrgMode;
          if (cu.timdMrg)
          {
            pu.intraDir[ CHANNEL_TYPE_LUMA ] = cu.timdMrgList[cu.timdMrg - 1][0];
            cu.timd = true;
    
    Keming Cao's avatar
    Keming Cao committed
          }
    #endif
    
    fan wang's avatar
    fan wang committed
    #if JVET_AB0155_SGPM
          cu.sgpm = uiBestPUMode.sgpmFlag;
          if (cu.sgpm)
          {
            CHECK(!bestSgpmMode, "mode not same");
    
    #if JVET_AG0152_SGPM_ITMP_IBC
              pu.intraDir[CHANNEL_TYPE_LUMA] = uiBestPUMode.sgpmMode0 >= SGPM_BV_START_IDX ? 0 : uiBestPUMode.sgpmMode0;
              pu.intraDir1[CHANNEL_TYPE_LUMA] = uiBestPUMode.sgpmMode1 >= SGPM_BV_START_IDX ? 0 : uiBestPUMode.sgpmMode1;
    #else
    
    fan wang's avatar
    fan wang committed
            pu.intraDir[CHANNEL_TYPE_LUMA]  = uiBestPUMode.sgpmMode0;
            pu.intraDir1[CHANNEL_TYPE_LUMA] = uiBestPUMode.sgpmMode1;
    
    fan wang's avatar
    fan wang committed
            cu.sgpmSplitDir                 = uiBestPUMode.sgpmSplitDir;
            cu.sgpmMode0                    = uiBestPUMode.sgpmMode0;
            cu.sgpmMode1                    = uiBestPUMode.sgpmMode1;
            cu.sgpmIdx                      = uiBestPUMode.sgpmIdx;
    
    #if JVET_AG0152_SGPM_ITMP_IBC
            cu.sgpmBv0                       = uiBestPUMode.sgpmBv0;
            cu.sgpmBv1                      = uiBestPUMode.sgpmBv1;
    
    #endif
    #if JVET_AJ0112_REGRESSION_SGPM
            cu.blendModel                   = uiBestPUMode.sgpmBlendModel;
    
    #if JVET_AB0157_TMRL
          cu.tmrlFlag = uiBestPUMode.mRefId >= MAX_REF_LINE_IDX;
          if(cu.tmrlFlag)
          {
            int tmrlListIdx = uiBestPUMode.mRefId - MAX_REF_LINE_IDX;
            cu.tmrlListIdx = tmrlListIdx;
    
            pu.multiRefIdx = m_tmrlList[tmrlListIdx].multiRefIdx;
            pu.intraDir[0] = m_tmrlList[tmrlListIdx].intraDir;
    
    #if JVET_AG0058_EIP
          cu.eipFlag = (uiBestPUMode.modeId >= EIP_IDX) && (uiBestPUMode.modeId < EIP_IDX + std::max(NUM_DERIVED_EIP, MAX_MERGE_EIP));
          if (cu.eipFlag)
          {
            cu.eipMerge = uiBestPUMode.mipTrFlg;
            pu.intraDir[0] = uiBestPUMode.modeId - EIP_IDX;
            cu.eipModel = cu.eipMerge ? m_eipMergeModel[pu.intraDir[0]] : m_eipModel[pu.intraDir[0]];
    
    #if JVET_AJ0082_MM_EIP
            cu.eipMmFlag = (!cu.eipMerge) && (pu.intraDir[0] >= m_numSigEip);
            if(cu.eipMmFlag)
            {
              pu.intraDir[0] -= m_numSigEip;
            }
            if (cu.eipModel.eipDimdMode == -1)
            {
              const auto modeIdx = cu.eipMmFlag ? (pu.intraDir[0] + m_numSigEip): pu.intraDir[0];
    
              if(cu.eipMerge)
              {
                PelBuf eipSaveBuf(m_eipMergePredBuf[modeIdx], pu.Y());
                cu.eipModel.eipDimdMode = m_eipMergeModel[modeIdx].eipDimdMode = deriveIpmForTransform(eipSaveBuf, cu
    #if JVET_AI0050_INTER_MTSS
                  , cu.dimdDerivedIntraDir2nd
    #endif
                );
                CHECK(modeIdx >= NUM_EIP_MERGE_SIGNAL, "modeIdx >= NUM_EIP_MERGE_SIGNAL");
              }
              else
              {
                PelBuf eipSaveBuf(m_eipPredBuf[modeIdx], pu.Y());
                cu.eipModel.eipDimdMode = m_eipModel[modeIdx].eipDimdMode = deriveIpmForTransform(eipSaveBuf, cu
    #if JVET_AI0050_INTER_MTSS
                  , cu.dimdDerivedIntraDir2nd
    #endif
                );
                CHECK(modeIdx >= NUM_DERIVED_EIP, "modeIdx >= NUM_DERIVED_EIP");
              }
            }
    #endif  
    
    #if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST 
          if (cu.tmpFlag)
          {
            cu.intraTmpDimdMode = intraTmpDimdMode;
          }
    
    #endif
    
    #if JVET_AC0105_DIRECTIONAL_PLANAR
          cu.plIdx = bestPlMode;
          if (cu.plIdx)
          {
            CHECK(pu.multiRefIdx > 0, "use of PL");
            CHECK(cu.mipFlag, "use of PL");
            CHECK(cu.dimd, "use of PL");
            CHECK(cu.tmpFlag, "use of PL");
            CHECK(cu.tmrlFlag, "use of PL");
            CHECK(cu.sgpm, "use of PL");
            CHECK(cu.timd, "use of PL");
          }
    #endif
    
    #if JVET_AJ0249_NEURAL_NETWORK_BASED
          cu.lfnstSecFlag = bestLfnstSecFlag;
    #endif
    
          if (cu.colorTransform)
          {
            CHECK(pu.intraDir[CHANNEL_TYPE_CHROMA] != DM_CHROMA_IDX, "chroma should use DM mode for adaptive color transform");
          }
    
      }
    
      //===== reset context models =====
      m_CABACEstimator->getCtx() = ctxStart;
    
    #if JVET_AD0120_LBCCP
    void IntraSearch::fillLmPredFiltList(PredictionUnit pu, const PelUnitBuf& lmPredFilt, int &lmPredFiltIdx, std::vector<lmPredFiltModeInfo> &miLmPredFiltList)
    {
      const TempCtx ctxStart(m_ctxCache, m_CABACEstimator->getCtx());
      const double  sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda() * FRAC_BITS_SCALE;
    
      int64_t          sad = 0, sadCb = 0, satdCb = 0, sadCr = 0, satdCr = 0;
      CodingStructure &cs = *(pu.cs);
      DistParam        distParamSadCb, distParamSatdCb;
      DistParam        distParamSadCr, distParamSatdCr;
      m_pcRdCost->setDistParam(distParamSadCb, cs.getOrgBuf(pu.Cb()), lmPredFilt.Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, false);
      m_pcRdCost->setDistParam(distParamSatdCb, cs.getOrgBuf(pu.Cb()), lmPredFilt.Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, true);
      m_pcRdCost->setDistParam(distParamSadCr, cs.getOrgBuf(pu.Cr()), lmPredFilt.Cr(),  pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, false);
      m_pcRdCost->setDistParam(distParamSatdCr, cs.getOrgBuf(pu.Cr()), lmPredFilt.Cr(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, true);
      distParamSadCb.applyWeight  = false;
      distParamSatdCb.applyWeight = false;
      distParamSadCr.applyWeight  = false;
      distParamSatdCr.applyWeight = false;
    
      sadCb  = distParamSadCb.distFunc(distParamSadCb) * 2;
      satdCb = distParamSatdCb.distFunc(distParamSatdCb);
      sad    = std::min(sadCb, satdCb);
      sadCr  = distParamSadCr.distFunc(distParamSadCr) * 2;
      satdCr = distParamSatdCr.distFunc(distParamSatdCr);
      sad += std::min(sadCr, satdCr);
    
      m_CABACEstimator->getCtx() = ctxStart;
      uint64_t fracModeBits      = xFracModeBitsIntra(pu, MMLM_CHROMA_IDX, CHANNEL_TYPE_CHROMA);
      double   cost              = (double) sad + (double) fracModeBits * sqrtLambdaForFirstPass;
    
      int cccmFlag = 0, cccmNoSubFlag = 0, glCccmFlag = 0, cccmMultiFilterIdx = 0;
    #if JVET_AA0057_CCCM
      cccmFlag = pu.cccmFlag;
    #endif
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING
      cccmNoSubFlag = pu.cccmNoSubFlag;
    #endif
    #if JVET_AC0054_GLCCCM
      glCccmFlag = pu.glCccmFlag;
    #endif
    #if JVET_AD0202_CCCM_MDF
      cccmMultiFilterIdx = pu.cccmMultiFilterIdx;
    #endif
      miLmPredFiltList.push_back({ lmPredFiltIdx, cccmFlag, cccmNoSubFlag, glCccmFlag, cccmMultiFilterIdx, cost });
      lmPredFiltIdx++;
    }
    #endif
    
    
    #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
    void IntraSearch::getPredForCCPMrgFusion(PredictionUnit& pu, PelBuf& predCb, PelBuf& predCr)
    {
      int            width = predCb.width;
      int            height = predCb.height;
    
      const int      scaleX = getComponentScaleX(COMPONENT_Cb, pu.chromaFormat);
      const int      scaleY = getComponentScaleY(COMPONENT_Cb, pu.chromaFormat);
      const UnitArea localUnitArea(pu.chromaFormat, Area(0, 0, width << scaleX, height << scaleY));
    
      PredictionUnit pu2 = pu;
    
      if (pu.ccpMergeFusionType == 0)
      {
        const int                         bitDepth = pu.cu->slice->getSPS()->getBitDepth(CHANNEL_TYPE_LUMA);
        static CccmModel cccmModelCb[2] = { CccmModel(CCCM_NUM_PARAMS, bitDepth), CccmModel(CCCM_NUM_PARAMS, bitDepth) };
        static CccmModel cccmModelCr[2] = { CccmModel(CCCM_NUM_PARAMS, bitDepth), CccmModel(CCCM_NUM_PARAMS, bitDepth) };
        pu2.cccmFlag = 1;
    #if JVET_AC0054_GLCCCM
        pu2.glCccmFlag = 0;
    #endif
    #if JVET_AD0202_CCCM_MDF
        pu2.cccmMultiFilterIdx = 0;
    #endif
    #if MMLM
        pu2.intraDir[1] = MMLM_CHROMA_IDX;
        static int modelThr = 0;
    
        modelThr = xCccmCalcRefAver(pu2);
        xCccmCalcModels(pu2, cccmModelCb[0], cccmModelCr[0], 1, modelThr);
        xCccmCalcModels(pu2, cccmModelCb[1], cccmModelCr[1], 2, modelThr);
    
        xCccmApplyModel(pu2, COMPONENT_Cb, cccmModelCb[0], 1, modelThr, predCb);
        xCccmApplyModel(pu2, COMPONENT_Cb, cccmModelCb[1], 2, modelThr, predCb);
    
        xCccmApplyModel(pu2, COMPONENT_Cr, cccmModelCr[0], 1, modelThr, predCr);
        xCccmApplyModel(pu2, COMPONENT_Cr, cccmModelCr[1], 2, modelThr, predCr);
    #else
        pu2.intraDir[1] = LM_CHROMA_IDX;
    
        xCccmCalcModels(pu2, cccmModelCb[0], cccmModelCr[0], 0, 0);
    
        xCccmApplyModel(pu2, COMPONENT_Cb, cccmModelCb[0], 0, 0, predCb);
        xCccmApplyModel(pu2, COMPONENT_Cr, cccmModelCr[0], 0, 0, predCr);
    #endif
      }
      else
      {
        pu.intraDir[1] = DIMD_CHROMA_IDX;
        initIntraPatternChType(*pu.cu, pu.blocks[COMPONENT_Cb]);
        initIntraPatternChType(*pu.cu, pu.blocks[COMPONENT_Cr]);
        predIntraAng(COMPONENT_Cb, predCb, pu);
        predIntraAng(COMPONENT_Cr, predCr, pu);
        pu.intraDir[1] = LM_CHROMA_IDX;
      }
    
    }
    void IntraSearch::xCalcCcpMrgPred(const PredictionUnit& pu, const ComponentID compID, PelBuf& piPred, PelBuf& piLm)
    {
      CHECK(compID == COMPONENT_Y, "");
    
      int width = piPred.width;
      int height = piPred.height;
      Pel* pelPred = piPred.buf;
      Pel* pelLm = piLm.buf;
    
      int  w0 = 2;
      int  w1 = 2;
      int  shift = 2;
    
      for (int y = 0; y < height; y++)
      {
        for (int x = 0; x < width; x++)
        {
          int blend = pelPred[x] * w0;
          blend += pelLm[x] * w1;
          blend += 2;
          pelLm[x] = (Pel)(blend >> shift);
        }
        pelPred += piPred.stride;
        pelLm += piLm.stride;
      }
    }
    #endif
    
    
    void IntraSearch::estIntraPredChromaQT( CodingUnit &cu, Partitioner &partitioner, const double maxCostAllowed 
    #if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                                          , InterPrediction* pcInterPred
    #endif
    )
    
    {
      const ChromaFormat format   = cu.chromaFormat;
      const uint32_t    numberValidComponents = getNumberValidComponents(format);
      CodingStructure &cs = *cu.cs;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStart  ( m_ctxCache, m_CABACEstimator->getCtx() );
    
      double    bestCostSoFar = maxCostAllowed;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
      bool      lumaUsesISP   = !cu.isSepTree() && cu.ispMode;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #else
      bool      lumaUsesISP = !CS::isDualITree(*cu.cs) && cu.ispMode;
    #endif
    
      PartSplit ispType       = lumaUsesISP ? CU::getISPType( cu, COMPONENT_Y ) : TU_NO_ISP;
      CHECK( cu.ispMode && bestCostSoFar < 0, "bestCostSoFar must be positive!" );
    
    #if JVET_AF0066_ENABLE_DBV_4_SINGLE_TREE
      bool singleTreeLumaIntraTmp = !CS::isDualITree(*cu.cs) && cu.tmpFlag;
    #endif
    
    #if JVET_AD0120_LBCCP
      int  bestCCInsideFilter = 0;
    #endif
    
      auto &pu = *cu.firstPU;
    
      {
        uint32_t       uiBestMode = 0;
        Distortion uiBestDist = 0;
        double     dBestCost = MAX_DOUBLE;
    
        int32_t bestBDPCMMode = 0;
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
        Distortion bestDist = ULLONG_MAX;
        int      decoderDerivedCcpModeBest = 0;
        int      bestDdNonLocalMergeFusion = 0;
    #endif
    
    Jani Lainema's avatar
    Jani Lainema committed
    #if JVET_AA0057_CCCM
        int      cccmModeBest = 0;
    
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING
        int      cccmNoSubBest = 0;
    #endif
    
    #if JVET_AC0054_GLCCCM
        int      glCccmBest = 0;
    #endif
    
    #if JVET_AE0100_BVGCCCM
        int      bvgCccmBest = 0;
    #endif
    
    #if JVET_AD0202_CCCM_MDF
        int      cccmMultiFilterIdxBest = 0;
    #endif
    
    Jani Lainema's avatar
    Jani Lainema committed
    #endif
    
    #if JVET_Z0050_CCLM_SLOPE
        CclmOffsets bestCclmOffsets = {};
        CclmOffsets satdCclmOffsetsBest[NUM_CHROMA_MODE];
        int64_t     satdCclmCosts      [NUM_CHROMA_MODE] = { 0 };
    #endif
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
    #if JVET_AA0126_GLM
        GlmIdc      bestGlmIdc = {};
        GlmIdc      satdGlmIdcBest     [NUM_CHROMA_MODE];
        int64_t     satdGlmCosts       [NUM_CHROMA_MODE] = { 0 };
    #endif
    
    #if JVET_Z0050_DIMD_CHROMA_FUSION
    
    #if JVET_AC0119_LM_CHROMA_FUSION
        uint8_t isChromaFusion = 0;
    #else
    
        bool isChromaFusion = false;
    
    #endif
    #if JVET_AD0188_CCP_MERGE
        int               bestNonAdjCCCM = 0;
        CCPModelCandidate ccpModelBest;
    
    #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
        int bestCcpMergeFusionFlag = 0;
        int bestCcpMergeFusionType = 0;
    #endif
    
    #if JVET_AJ0081_CHROMA_TMRL
        bool bestChromaTmrlFlag = false;
        int bestChromaTmrlIdx = 0;
    #endif
    
          int32_t  uiMinMode = 0;
          int32_t  uiMaxMode = NUM_CHROMA_MODE;
    
          //----- check chroma modes -----
          uint32_t chromaCandModes[ NUM_CHROMA_MODE ];
          PU::getIntraChromaCandModes( pu, chromaCandModes );
    
    #if JVET_Z0050_DIMD_CHROMA_FUSION && ENABLE_DIMD
    
    #if JVET_AC0094_REF_SAMPLES_OPT
        if (!CS::isDualITree(*cu.cs))
        {
          const CompArea areaCb = pu.Cb();
          const CompArea areaCr = pu.Cr();
          const CompArea lumaArea = CompArea(COMPONENT_Y, pu.chromaFormat, areaCb.lumaPos(), recalcSize(pu.chromaFormat, CHANNEL_TYPE_CHROMA, CHANNEL_TYPE_LUMA, areaCb.size()));//needed for correct pos/size (4x4 Tus)
          IntraPrediction::deriveDimdChromaMode(cs.picture->getRecoBuf(lumaArea), cs.picture->getRecoBuf(areaCb), cs.picture->getRecoBuf(areaCr), lumaArea, areaCb, areaCr, *pu.cu);
        }
        if (PU::getCoLocatedIntraLumaMode(*cu.firstPU) == cu.dimdChromaMode)
        {
          if (cu.dimdChromaMode == cu.dimdChromaModeSecond)
          {
            cu.dimdChromaMode = DC_IDX;
          }
          else
          {
            cu.dimdChromaMode = cu.dimdChromaModeSecond;
          }
        }
    #else
    
          // derive DIMD chroma mode
          CompArea areaCb = pu.Cb();
          CompArea areaCr = pu.Cr();
          CompArea lumaArea = CompArea(COMPONENT_Y, pu.chromaFormat, areaCb.lumaPos(), recalcSize(pu.chromaFormat, CHANNEL_TYPE_CHROMA, CHANNEL_TYPE_LUMA, areaCb.size()));//needed for correct pos/size (4x4 Tus)
          IntraPrediction::deriveDimdChromaMode(cs.picture->getRecoBuf(lumaArea), cs.picture->getRecoBuf(areaCb), cs.picture->getRecoBuf(areaCr), lumaArea, areaCb, areaCr, *pu.cu);
    #endif
    
    #if JVET_AH0136_CHROMA_REORDERING && ENABLE_DIMD && JVET_Z0050_DIMD_CHROMA_FUSION
          if (!cu.slice->getSPS()->getUseDimd() && cu.cs->sps->getUseChromaReordering())
          {
            uiMaxMode--;
          }
    #endif
    
    #if JVET_AC0071_DBV
          if (PU::hasChromaBvFlag(pu))
          {
            PU::deriveChromaBv(pu);
          }
          else
          {
            uiMaxMode--;
          }
    #endif
    
    
          // create a temporary CS
          CodingStructure &saveCS = *m_pSaveCS[0];
          saveCS.pcv      = cs.pcv;
          saveCS.picture  = cs.picture;
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #if JVET_Z0118_GDR
          saveCS.m_pt = cs.m_pt;
    #endif
    
          saveCS.area.repositionTo( cs.area );
          saveCS.clearTUs();
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
          if( !cu.isSepTree() && cu.ispMode )
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #else
          if (!CS::isDualITree(cs) && cu.ispMode)
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
          if( cu.isSepTree() )
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #else
          if (CS::isDualITree(cs))
    #endif
    
            if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs
    #if JVET_AI0087_BTCUS_RESTRICTION
              , false, false
    #endif
            ) )
    
            {
              partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
    
              do
              {
                cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType ).depth = partitioner.currTrDepth;
              } while( partitioner.nextPart( cs ) );
    
              partitioner.exitCurrSplit();
            }
            else
            cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType );
          }
    
          std::vector<TransformUnit*> orgTUs;
    
    
          if( lumaUsesISP )
          {
            CodingUnit& auxCU = saveCS.addCU( cu, partitioner.chType );
            auxCU.ispMode = cu.ispMode;
            saveCS.sps = cu.cs->sps;
            saveCS.addPU( *cu.firstPU, partitioner.chType );
          }
    
    
    
          // create a store for the TUs
          for( const auto &ptu : cs.tus )
          {
            // for split TUs in HEVC, add the TUs without Chroma parts for correct setting of Cbfs
    
            if( lumaUsesISP || pu.contains( *ptu, CHANNEL_TYPE_CHROMA ) )
    
            {
              saveCS.addTU( *ptu, partitioner.chType );
              orgTUs.push_back( ptu );
            }
          }
    
          // SATD pre-selecting.
          int satdModeList[NUM_CHROMA_MODE];
          int64_t satdSortedCost[NUM_CHROMA_MODE];
          for (int i = 0; i < NUM_CHROMA_MODE; i++)
          {
            satdSortedCost[i] = 0; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
            satdModeList[i] = 0;
          }
    
    #if JVET_Z0050_DIMD_CHROMA_FUSION && ENABLE_DIMD
    
    #if JVET_AH0136_CHROMA_REORDERING
          bool modeIsEnable[NUM_INTRA_MODE + 3 + 9]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 3 + 9; i++)
          {
            modeIsEnable[i] = 1;
          }
    #else
    
          bool modeIsEnable[NUM_INTRA_MODE + 3]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 3; i++)
          {
            modeIsEnable[i] = 1;
          }
    
    #else
          bool modeIsEnable[NUM_INTRA_MODE + 2]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 2; i++)
          {
            modeIsEnable[i] = 1;
          }
    #endif
    #else
    #if JVET_AC0071_DBV
    
          bool modeIsEnable[NUM_INTRA_MODE + 2]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 2; i++)
          {
            modeIsEnable[i] = 1;
          }
    #else
    
          bool modeIsEnable[NUM_INTRA_MODE + 1]; // use intra mode idx to check whether enable
          for (int i = 0; i < NUM_INTRA_MODE + 1; i++)
          {
            modeIsEnable[i] = 1;
          }
    
          DistParam distParamSad;
          DistParam distParamSatd;
    
          pu.intraDir[1] = MDLM_L_IDX; // temporary assigned, just to indicate this is a MDLM mode. for luma down-sampling operation.
    
          initIntraPatternChType(cu, pu.Cb());
          initIntraPatternChType(cu, pu.Cr());
          xGetLumaRecPixels(pu, pu.Cb());
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
    
    #if JVET_AA0126_GLM
          if ( PU::isLMCModeEnabled( pu, LM_CHROMA_IDX ) && PU::hasGlmFlag( pu, LM_CHROMA_IDX ) )
          {
    
    #if JVET_AB0092_GLM_WITH_LUMA && JVET_AB0174_CCCM_DIV_FREE
            xGlmSetLumaRefValue(pu, pu.Cb());
    #endif
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
            // Generate all GLM templates at encoder
            xGetLumaRecPixelsGlmAll(pu, pu.Cb());
    
            pu.intraDir[1] = LM_CHROMA_IDX;
            xGetLumaRecPixels(pu, pu.Cb());
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
    
            for ( int mode = LM_CHROMA_IDX; mode <= MMLM_T_IDX; mode++ )
            {
              satdGlmIdcBest[mode - LM_CHROMA_IDX].setAllZero();
              
    
    #if JVET_AB0092_GLM_WITH_LUMA
    
    #if JVET_AI0136_ADAPTIVE_DUAL_TREE
              CodedCUInfo* relatedCU = ((EncModeCtrlMTnoRQT *)m_modeCtrl)->getBlkInfoPtr(partitioner.currArea());
              if (relatedCU && PU::hasGlmFlag(pu, mode) && !relatedCU->skipGLM)
    #else
    
              CodedCUInfo& relatedCU = ((EncModeCtrlMTnoRQT *)m_modeCtrl)->getBlkInfo(partitioner.currArea());
              if (PU::hasGlmFlag(pu, mode) && !relatedCU.skipGLM)
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
              if ( PU::hasGlmFlag( pu, mode ) )
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
              {
    
    #if !JVET_AB0092_GLM_WITH_LUMA
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
                for ( int comp = COMPONENT_Cb; comp <= COMPONENT_Cr; comp++ )
                {
                  ComponentID       compID = ComponentID( comp );
    
    Vadim Seregin's avatar
    Vadim Seregin committed
                  ComponentID       compID = COMPONENT_Cb;
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
                  int              idcBest = 0;
                  int64_t         satdBest = 0;
                  GlmIdc&         idcsBest = satdGlmIdcBest[mode - LM_CHROMA_IDX];
                  
                  pu.intraDir[1] = mode;
                  pu.glmIdc.setAllZero();
    
                  xFindBestGlmIdcSATD(pu, compID, idcBest, satdBest );
    
                  idcsBest.setIdc(compID, 0, idcBest);
                  idcsBest.setIdc(compID, 1, idcBest);
    
    
    #if JVET_AB0092_GLM_WITH_LUMA
                  idcsBest.setIdc(COMPONENT_Cr, 0, idcBest);
                  idcsBest.setIdc(COMPONENT_Cr, 1, idcBest);
    #endif
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
                  
                  satdGlmCosts[mode - LM_CHROMA_IDX] += satdBest; // Summing up Cb and Cr cost
    
    #if !JVET_AB0092_GLM_WITH_LUMA
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
                }
                
                if ( !satdGlmIdcBest[0].isActive() )
                {
                  break;
                }
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
              }
            }
          }
    
          pu.glmIdc.setAllZero();
    #endif
    
          
    #if JVET_Z0050_CCLM_SLOPE
          if ( PU::isLMCModeEnabled( pu, LM_CHROMA_IDX ) && PU::hasCclmDeltaFlag( pu, LM_CHROMA_IDX ) )
          {
            // Fill luma reference buffer for the two-sided CCLM
            pu.intraDir[1] = LM_CHROMA_IDX;
            xGetLumaRecPixels(pu, pu.Cb());
    
            for ( int mode = LM_CHROMA_IDX; mode <= MDLM_T_IDX; mode++ )
            {
              satdCclmOffsetsBest[mode - LM_CHROMA_IDX].setAllZero();
              
              if ( PU::hasCclmDeltaFlag( pu, mode ) )
              {
                for ( int comp = COMPONENT_Cb; comp <= COMPONENT_Cr; comp++ )
                {
                  ComponentID       compID = ComponentID( comp );
                  int            deltaBest = 0;
                  int64_t         satdBest = 0;
                  CclmOffsets& offsetsBest = satdCclmOffsetsBest[mode - LM_CHROMA_IDX];
                  
                  pu.intraDir[1] = mode;
                  pu.cclmOffsets.setAllZero();
    
                  xFindBestCclmDeltaSlopeSATD(pu, compID, 0, deltaBest, satdBest );
    
                  offsetsBest.setOffset(compID, 0, deltaBest);
    
    #if MMLM
                  if ( PU::isMultiModeLM( mode ) )
                  {
                    // Set best found values for the first model to get a matching second model
                    pu.cclmOffsets.setOffsets(offsetsBest.cb0, offsetsBest.cr0, 0, 0);
    
                    xFindBestCclmDeltaSlopeSATD(pu, compID, 1, deltaBest, satdBest );
    
                    offsetsBest.setOffset(compID, 1, deltaBest);
                  }
    #endif
    
                  satdCclmCosts[mode - LM_CHROMA_IDX] += satdBest; // Summing up Cb and Cr cost
                }
              }
            }
          }
    
          pu.cclmOffsets.setAllZero();
    #endif
    
    #if JVET_AD0120_LBCCP && MMLM
          std::vector<lmPredFiltModeInfo> miLmPredFiltList;
          miLmPredFiltList.clear();
    #if JVET_AD0188_CCP_MERGE
          CCPModelCandidate ccpCandlmPredFilt[LBCCP_FILTER_MMLMNUM];
    #endif
          PelUnitBuf lmPredFilterStorage[LBCCP_FILTER_MMLMNUM];
          const UnitArea tmpUnitArea(pu.chromaFormat, Area(0, 0, (pu.Cb().width) << getChannelTypeScaleX(CHANNEL_TYPE_CHROMA, pu.chromaFormat), (pu.Cb().height) << getChannelTypeScaleY(CHANNEL_TYPE_CHROMA, pu.chromaFormat)));
          for (uint32_t i = 0; i < LBCCP_FILTER_MMLMNUM; i++)
          {
            lmPredFilterStorage[i] = m_lmPredFiltStorage[i].getBuf(tmpUnitArea);
    #if JVET_AD0188_CCP_MERGE
            ccpCandlmPredFilt[i] = {};
    #endif
          }
          int lmPredFiltIdx = 0;
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if MMLM
          m_encPreRDRun = true;
    #endif
    
          for (int idx = uiMinMode; idx <= uiMaxMode - 1; idx++)
          {
            int mode = chromaCandModes[idx];
            satdModeList[idx] = mode;
            if (PU::isLMCMode(mode) && !PU::isLMCModeEnabled(pu, mode))
            {
              continue;
            }
    
    #if JVET_AH0136_CHROMA_REORDERING
    
    #if JVET_AI0136_ADAPTIVE_DUAL_TREE
            if (CS::isDualITree(*pu.cs) && cu.cs->slice->isIntra() && cu.cs->sps->getUseChromaReordering())
    #else
    
            if (CS::isDualITree(*pu.cs) && cu.cs->sps->getUseChromaReordering())
    
            {
              if ((mode == LM_CHROMA_IDX) || mode == pu.cu->chromaList[0] || mode == pu.cu->chromaList[1]) 
              {
                continue;
              }
    #if ENABLE_DIMD && JVET_Z0050_DIMD_CHROMA_FUSION
    #if JVET_AC0071_DBV
              if ((cu.slice->getSPS()->getUseDimd() && (mode == pu.cu->chromaList[2] || (PU::hasChromaBvFlag(pu) && mode == pu.cu->chromaList[3]))) || (!cu.slice->getSPS()->getUseDimd() && (PU::hasChromaBvFlag(pu) && mode == pu.cu->chromaList[2])))
              {
                continue;
              }
    #else
              if ((cu.slice->getSPS()->getUseDimd() && mode == pu.cu->chromaList[2]))
              {
                continue;
              }
    
              if (PU::hasChromaBvFlag(pu) && mode == pu.cu->chromaList[2])
              {
                continue;
              }
    #endif
    
    #endif
              if ((mode == LM_CHROMA_IDX) || (mode == PLANAR_IDX) || (mode == DM_CHROMA_IDX)
    #if JVET_Z0050_DIMD_CHROMA_FUSION && ENABLE_DIMD
                || (mode == DIMD_CHROMA_IDX)
    #endif
    #if JVET_AC0071_DBV
                || (mode == DBV_CHROMA_IDX)
    #endif
                ) // only pre-check regular modes and MDLM modes, not including DM, DIMD, Planar, and LM
              {
                continue;
              }
    #if JVET_AH0136_CHROMA_REORDERING
    
            pu.intraDir[1] = mode; // temporary assigned, for SATD checking.
    
    
            int64_t sad = 0;
            int64_t sadCb = 0;
            int64_t satdCb = 0;
            int64_t sadCr = 0;
            int64_t satdCr = 0;
    
            CodingStructure& cs = *(pu.cs);
    
    #if JVET_AD0120_LBCCP && JVET_AD0188_CCP_MERGE
            pu.curCand = {};
    #endif
    
    
            CompArea areaCb = pu.Cb();
            PelBuf orgCb = cs.getOrgBuf(areaCb);
            PelBuf predCb = cs.getPredBuf(areaCb);
    
            m_pcRdCost->setDistParam(distParamSad, orgCb, predCb, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, false);
            m_pcRdCost->setDistParam(distParamSatd, orgCb, predCb, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, true);
            distParamSad.applyWeight = false;
            distParamSatd.applyWeight = false;
    
            if (PU::isLMCMode(mode))
            {
              predIntraChromaLM(COMPONENT_Cb, predCb, pu, areaCb, mode);
            }
            else
            {
    
    #if JVET_AH0136_CHROMA_REORDERING && JVET_AC0071_DBV
    
    #if JVET_AI0136_ADAPTIVE_DUAL_TREE
              if (cu.cs->sps->getUseChromaReordering() && CS::isDualITree(cs) && cs.slice->isIntra() && PU::isDbvMode(mode))
    #else
    
              if (cu.cs->sps->getUseChromaReordering() && CS::isDualITree(cs) && PU::isDbvMode(mode))
    
                if (cu.mvs[mode - DBV_CHROMA_IDX] == Mv())
                {
                  satdSortedCost[idx] = INT64_MAX;
                  continue;
                }
    
                pu.bv = cu.bvs[mode - DBV_CHROMA_IDX];
                pu.mv[0] = cu.mvs[mode - DBV_CHROMA_IDX];
                pu.cu->rribcFlipType = cu.rribcTypes[mode - DBV_CHROMA_IDX];
                predIntraDbv(COMPONENT_Cb, predCb, pu, pcInterPred);
                if (pu.cu->rribcFlipType)
                {
                  predCb.flipSignal(pu.cu->rribcFlipType == 1);
                }
                pu.bv.setZero();
                pu.mv[0].setZero();
                pu.cu->rribcFlipType = 0;
              }
              else
              {
    #endif
                initPredIntraParams(pu, pu.Cb(), *pu.cs->sps);
                predIntraAng(COMPONENT_Cb, predCb, pu);
    #if JVET_AH0136_CHROMA_REORDERING
              }
    #endif
    
            sadCb = distParamSad.distFunc(distParamSad) * 2;
            satdCb = distParamSatd.distFunc(distParamSatd);
            sad += std::min(sadCb, satdCb);
    
            CompArea areaCr = pu.Cr();
            PelBuf orgCr = cs.getOrgBuf(areaCr);
            PelBuf predCr = cs.getPredBuf(areaCr);
    
            m_pcRdCost->setDistParam(distParamSad, orgCr, predCr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, false);
            m_pcRdCost->setDistParam(distParamSatd, orgCr, predCr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, true);
            distParamSad.applyWeight = false;
            distParamSatd.applyWeight = false;
    
            if (PU::isLMCMode(mode))
            {
              predIntraChromaLM(COMPONENT_Cr, predCr, pu, areaCr, mode);
            }
            else
            {
    
    #if JVET_AH0136_CHROMA_REORDERING && JVET_AC0071_DBV
    
    #if JVET_AI0136_ADAPTIVE_DUAL_TREE
              if (cu.cs->sps->getUseChromaReordering() && CS::isDualITree(cs) && cs.slice->isIntra() && PU::isDbvMode(mode))
    #else
    
              if (cu.cs->sps->getUseChromaReordering() && CS::isDualITree(cs) && PU::isDbvMode(mode))
    
              {
                pu.bv = cu.bvs[mode - DBV_CHROMA_IDX];
                pu.mv[0] = cu.mvs[mode - DBV_CHROMA_IDX];
                pu.cu->rribcFlipType = cu.rribcTypes[mode - DBV_CHROMA_IDX];
                predIntraDbv(COMPONENT_Cr, predCr, pu, pcInterPred);
                if (pu.cu->rribcFlipType)
                {
                  predCr.flipSignal(pu.cu->rribcFlipType == 1);
                }
                pu.bv.setZero();
                pu.mv[0].setZero();
                pu.cu->rribcFlipType = 0;
              }
              else