Skip to content
Snippets Groups Projects
IntraSearch.cpp 442 KiB
Newer Older
  • Learn to ignore specific revisions
  • #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
            pu.ccpMergeFusionFlag = 0;
            pu.ccpMergeFusionType = 0;
    #endif
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
            if (pu.cu->slice->getSPS()->getUseDdCcpFusion())
            {
              pu.idxNonLocalCCP = 1;
              for (int rdIdx = 0; rdIdx < finalNumRDtestFusion; rdIdx++)
              {
                int chromaIntraMode = LM_CHROMA_IDX;
                pu.ddNonLocalCCPFusion = m_ddCcpMergeFusionModeIndex[rdIdx] + 1;
                pu.curCand = candList[FusionList[2 * m_ddCcpMergeFusionModeIndex[rdIdx]]];
    
                // Original RD check code replicated from above
                cs.setDecomp(pu.Cb(), false);
                cs.dist = baseDist;
                //----- restore context models -----
                m_CABACEstimator->getCtx() = ctxStart;
    
                //----- chroma coding -----
                pu.intraDir[1] = chromaIntraMode;
    #if JVET_AB0143_CCCM_TS
                xRecurIntraChromaCodingQT(cs, partitioner, bestCostSoFar, ispType, m_ddCcpFusionStorageTemp[rdIdx]);
    #else
                xRecurIntraChromaCodingQT(cs, partitioner, bestCostSoFar, ispType);
    #endif
                if (lumaUsesISP && cs.dist == MAX_UINT)
                {
                  continue;
                }
    
                if (cs.sps->getTransformSkipEnabledFlag())
                {
                  m_CABACEstimator->getCtx() = ctxStart;
                }
    
                uint64_t   fracBits = xGetIntraFracBitsQT(cs, partitioner, false, true, -1, ispType);
                Distortion uiDist = cs.dist;
                double     dCost = m_pcRdCost->calcRdCost(fracBits, uiDist - baseDist);
    
                //----- compare -----
                if (dCost < dBestCost)
                {
                  if (uiDist < bestDist)
                  {
                    bestDist = uiDist;
                  }
                  if (lumaUsesISP && dCost < bestCostSoFar)
                  {
                    bestCostSoFar = dCost;
                  }
                  for (uint32_t i = getFirstComponentOfChannel(CHANNEL_TYPE_CHROMA); i < numberValidComponents; i++)
                  {
                    const CompArea &area = pu.blocks[i];
    
                    saveCS.getRecoBuf(area).copyFrom(cs.getRecoBuf(area));
    #if KEEP_PRED_AND_RESI_SIGNALS
                    saveCS.getPredBuf(area).copyFrom(cs.getPredBuf(area));
                    saveCS.getResiBuf(area).copyFrom(cs.getResiBuf(area));
    #endif
                    saveCS.getPredBuf(area).copyFrom(cs.getPredBuf(area));
                    cs.picture->getPredBuf(area).copyFrom(cs.getPredBuf(area));
    #if !JVET_AB0143_CCCM_TS
                    cs.picture->getRecoBuf(area).copyFrom(cs.getRecoBuf(area));
    #endif
    
                    for (uint32_t j = 0; j < saveCS.tus.size(); j++)
                    {
                      saveCS.tus[j]->copyComponentFrom(*orgTUs[j], area.compID);
                    }
                  }
    
                  dBestCost = dCost;
                  uiBestDist = uiDist;
    
                  uiBestMode = chromaIntraMode;
    
                  bestBDPCMMode = cu.bdpcmModeChroma;
    #if JVET_Z0050_DIMD_CHROMA_FUSION
                  isChromaFusion = pu.isChromaFusion;
    #endif
    #if JVET_Z0050_CCLM_SLOPE
                  bestCclmOffsets = pu.cclmOffsets;
    #endif
                  cccmModeBest = pu.cccmFlag;
    #if JVET_AA0126_GLM
                  bestGlmIdc = pu.glmIdc;
    #endif
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING
                  cccmNoSubBest = pu.cccmNoSubFlag;
    #endif
    #if JVET_AC0054_GLCCCM
                  glCccmBest = pu.glCccmFlag;
    #endif
    #if JVET_AD0202_CCCM_MDF
                  cccmMultiFilterIdxBest = pu.cccmMultiFilterIdx;
    
    #endif
    #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
                  bestCCInsideFilter = pu.ccInsideFilter;
                  bestCcpMergeFusionFlag = pu.ccpMergeFusionFlag;
                  bestCcpMergeFusionType = pu.ccpMergeFusionType;
    
    #endif
                  bestNonAdjCCCM = pu.idxNonLocalCCP;
                  ccpModelBest = pu.curCand;
                  bestDdNonLocalMergeFusion = pu.ddNonLocalCCPFusion;
                }
              }
            }
    #endif
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
          pu.ddNonLocalCCPFusion = 0;
    #endif
    
    Jani Lainema's avatar
    Jani Lainema committed
    #endif
    
            
    #if JVET_AE0100_BVGCCCM
          pu.bvgCccmFlag = 0;
          if (PU::hasBvgCccmFlag(pu))
          {
            bool validBv = false;
            PU::getBvgCccmCands(pu, validBv);
            if (validBv)
            {
              for (int idx = 0; idx < 2; idx++)
              {
                int chromaIntraModeInCCCM = idx == 0 ? LM_CHROMA_IDX : MMLM_CHROMA_IDX;
                if (!PU::cccmSingleModeAvail(pu, LM_CHROMA_IDX))
                {
                  continue;
                }
                if (!PU::bvgCccmMultiModeAvail(pu, chromaIntraModeInCCCM) )
                {
                  continue;
                }
                pu.bvgCccmFlag = 1;
                pu.cccmFlag = 1;
                pu.intraDir[1] = chromaIntraModeInCCCM;
                pu.cccmNoSubFlag = 0;
                pu.glCccmFlag = 0;
                pu.isChromaFusion = 0;
                pu.cccmMultiFilterIdx = 0;
                pu.idxNonLocalCCP = 0;
                pu.ccInsideFilter = 0;
    #if JVET_AD0188_CCP_MERGE
                pu.curCand = {};
    #endif
                // Original RD check code replicated from above
                cs.setDecomp(pu.Cb(), false);
                cs.dist = baseDist;
                //----- restore context models -----
                m_CABACEstimator->getCtx() = ctxStart;
                xRecurIntraChromaCodingQT(cs, partitioner, bestCostSoFar, ispType);
                if (lumaUsesISP && cs.dist == MAX_UINT)
                {
                  continue;
                }
                if (cs.sps->getTransformSkipEnabledFlag())
                {
                  m_CABACEstimator->getCtx() = ctxStart;
                }
                uint64_t fracBits = xGetIntraFracBitsQT(cs, partitioner, false, true, -1, ispType);
                Distortion uiDist = cs.dist;
                double    dCost = m_pcRdCost->calcRdCost(fracBits, uiDist - baseDist);
                //----- compare -----
                if (dCost < dBestCost)
                {
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
                  if (uiDist < bestDist)
                  {
                    bestDist = uiDist;
                  }
    #endif
    
                  if (lumaUsesISP && dCost < bestCostSoFar)
                  {
                    bestCostSoFar = dCost;
                  }
                  for (uint32_t i = getFirstComponentOfChannel(CHANNEL_TYPE_CHROMA); i < numberValidComponents; i++)
                  {
                    const CompArea& area = pu.blocks[i];
                    
                    saveCS.getRecoBuf(area).copyFrom(cs.getRecoBuf(area));
    #if KEEP_PRED_AND_RESI_SIGNALS
                    saveCS.getPredBuf(area).copyFrom(cs.getPredBuf(area));
                    saveCS.getResiBuf(area).copyFrom(cs.getResiBuf(area));
    #endif
                    saveCS.getPredBuf(area).copyFrom(cs.getPredBuf(area));
                    cs.picture->getPredBuf(area).copyFrom(cs.getPredBuf(area));
    #if !JVET_AB0143_CCCM_TS
                    cs.picture->getRecoBuf(area).copyFrom(cs.getRecoBuf(area));
    #endif
                    
                    for (uint32_t j = 0; j < saveCS.tus.size(); j++)
                    {
                      saveCS.tus[j]->copyComponentFrom(*orgTUs[j], area.compID);
                    }
                  }
                  
                  dBestCost = dCost;
                  uiBestDist = uiDist;
    #if JVET_AB0143_CCCM_TS
                  uiBestMode = chromaIntraModeInCCCM;
    #else
                  uiBestMode = chromaIntraMode;
    #endif
                  bestBDPCMMode = cu.bdpcmModeChroma;
    #if JVET_Z0050_DIMD_CHROMA_FUSION
                  isChromaFusion = pu.isChromaFusion;
    #endif
    #if JVET_Z0050_CCLM_SLOPE
                  bestCclmOffsets = pu.cclmOffsets;
    #endif
                  cccmModeBest = pu.cccmFlag;
                  
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING
                  cccmNoSubBest = pu.cccmNoSubFlag;
    #endif
    #if JVET_AC0054_GLCCCM
                  glCccmBest = pu.glCccmFlag;
    #endif
                  bvgCccmBest = pu.bvgCccmFlag;
                  cccmModeBest = pu.cccmFlag;
    #if JVET_AD0202_CCCM_MDF
                  cccmMultiFilterIdxBest = pu.cccmMultiFilterIdx;
    #endif
                  bestCCInsideFilter = pu.ccInsideFilter;
    
    #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
                  bestCcpMergeFusionFlag = pu.ccpMergeFusionFlag;
                  bestCcpMergeFusionType = pu.ccpMergeFusionType;
    #endif
    
                  bestNonAdjCCCM = pu.idxNonLocalCCP;
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
                  bestDdNonLocalMergeFusion = pu.ddNonLocalCCPFusion;
    #endif
    
                  ccpModelBest   = pu.curCand;
                }
              }
            }
          }
          pu.bvgCccmFlag = 0;
          pu.cccmFlag = 0;
    #endif
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
          if (PU::hasDecoderDerivedCCP(pu))
          {
            if (!m_skipDdCcpListConstruction || !(cs.slice->getSliceType() == I_SLICE))
            {
              m_decoderDerivedCcpList.clear();
              pu.cccmFlag = 1;
              m_mmlmThreshold2 = xCccmCalcRefAver(pu, 2);
              decoderDerivedCcp(pu, m_decoderDerivedCcpList);
              m_skipDdCcpListConstruction = true;
            }
            int numDdccpModes = int(m_decoderDerivedCcpList.size());
    
            for (int idx = 0; idx < std::min(numDdccpModes, 1); idx++)
            {
              pu.decoderDerivedCcpMode = idx + 1;
              pu.intraDir[1] = m_decoderDerivedCcpList[idx].lmIndex;
    #if JVET_AA0057_CCCM
              pu.cccmFlag = m_decoderDerivedCcpList[idx].isCccm;
    #endif
    #if JVET_AC0054_GLCCCM
              pu.glCccmFlag = m_decoderDerivedCcpList[idx].isGlcccm;
    #endif
    #if JVET_AD0120_LBCCP
              pu.ccInsideFilter = m_decoderDerivedCcpList[idx].isInsideFilter;
    #endif
    #if JVET_AD0188_CCP_MERGE
              pu.curCand = m_decoderDerivedCcpList[idx].ddccpCand;
    #endif
    
              // Original RD check code replicated from above
              cs.setDecomp(pu.Cb(), false);
              cs.dist = baseDist;
              //----- restore context models -----
              m_CABACEstimator->getCtx() = ctxStart;
    
              //----- chroma coding -----
              xRecurIntraChromaCodingQT(cs, partitioner, bestCostSoFar, ispType, (firstTransformDdccp || !(cs.slice->getSliceType() == I_SLICE)) ? UnitBuf<Pel>() : m_ddCcpStorageTemp
    #if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                , pcInterPred
    #endif
              );
              if (lumaUsesISP && cs.dist == MAX_UINT)
              {
                continue;
              }
    
              if (cs.sps->getTransformSkipEnabledFlag())
              {
                m_CABACEstimator->getCtx() = ctxStart;
              }
    
              uint64_t   fracBits = xGetIntraFracBitsQT(cs, partitioner, false, true, -1, ispType);
              Distortion uiDist = cs.dist;
              double     dCost = m_pcRdCost->calcRdCost(fracBits, uiDist - baseDist);
    
              //----- compare -----
              if ((dCost < dBestCost) && ((cs.slice->getSliceType() == I_SLICE) || (uiDist < 1.1 * bestDist)))
              {
                if (lumaUsesISP && dCost < bestCostSoFar)
                {
                  bestCostSoFar = dCost;
                }
                for (uint32_t i = getFirstComponentOfChannel(CHANNEL_TYPE_CHROMA); i < numberValidComponents; i++)
                {
                  const CompArea &area = pu.blocks[i];
    
                  saveCS.getRecoBuf(area).copyFrom(cs.getRecoBuf(area));
    #if KEEP_PRED_AND_RESI_SIGNALS
                  saveCS.getPredBuf(area).copyFrom(cs.getPredBuf(area));
                  saveCS.getResiBuf(area).copyFrom(cs.getResiBuf(area));
    #endif
                  saveCS.getPredBuf(area).copyFrom(cs.getPredBuf(area));
                  cs.picture->getPredBuf(area).copyFrom(cs.getPredBuf(area));
    #if !JVET_AB0143_CCCM_TS
                  cs.picture->getRecoBuf(area).copyFrom(cs.getRecoBuf(area));
    #endif
    
                  for (uint32_t j = 0; j < saveCS.tus.size(); j++)
                  {
                    saveCS.tus[j]->copyComponentFrom(*orgTUs[j], area.compID);
                  }
                }
    
                dBestCost = dCost;
                uiBestDist = uiDist;
    
                uiBestMode = pu.intraDir[1];
                bestBDPCMMode = cu.bdpcmModeChroma;
    #if JVET_Z0050_DIMD_CHROMA_FUSION
                isChromaFusion = 0;
    #endif
                decoderDerivedCcpModeBest = pu.decoderDerivedCcpMode;
    #if JVET_AA0057_CCCM
                cccmModeBest = 0;
    #endif
    #if JVET_AD0202_CCCM_MDF
                cccmMultiFilterIdxBest = 0;
    #endif
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING
                cccmNoSubBest = 0;
    #endif
    #if JVET_AC0054_GLCCCM
                glCccmBest = 0;
    #endif
    #if JVET_AE0100_BVGCCCM
                bvgCccmBest = 0;
    #endif
    #if JVET_AD0120_LBCCP
                bestCCInsideFilter = 0;
    #endif
    #if JVET_Z0050_CCLM_SLOPE
                bestCclmOffsets = pu.cclmOffsets;
    #endif
    #if JVET_AA0126_GLM
                bestGlmIdc = pu.glmIdc;
    #endif
    #if JVET_AD0188_CCP_MERGE
                ccpModelBest = pu.curCand;
                bestDdNonLocalMergeFusion = pu.ddNonLocalCCPFusion;
    
    #endif
    #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
                bestCcpMergeFusionFlag = pu.ccpMergeFusionFlag;
                bestCcpMergeFusionType = pu.ccpMergeFusionType;
    
    #endif
                bestNonAdjCCCM = pu.idxNonLocalCCP;
              }
            }
            pu.decoderDerivedCcpMode = 0;
    #if JVET_AA0057_CCCM
            pu.cccmFlag = 0;
    #endif
    #if JVET_AC0054_GLCCCM
            pu.glCccmFlag = 0;
    #endif
    #if JVET_AD0120_LBCCP
            pu.ccInsideFilter = 0;
    #endif
    #if JVET_AD0188_CCP_MERGE
            pu.curCand = {};
    #endif
          }
    #endif
    
    
          for( uint32_t i = getFirstComponentOfChannel( CHANNEL_TYPE_CHROMA ); i < numberValidComponents; i++ )
          {
            const CompArea &area = pu.blocks[i];
    
            cs.getRecoBuf         ( area ).copyFrom( saveCS.getRecoBuf( area ) );
    #if KEEP_PRED_AND_RESI_SIGNALS
            cs.getPredBuf         ( area ).copyFrom( saveCS.getPredBuf( area ) );
            cs.getResiBuf         ( area ).copyFrom( saveCS.getResiBuf( area ) );
    #endif
    
    Taoran Lu's avatar
    Taoran Lu committed
            cs.getPredBuf         ( area ).copyFrom( saveCS.getPredBuf( area ) );
            cs.picture->getPredBuf( area ).copyFrom( cs.getPredBuf    ( area ) );
    
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #if JVET_Z0118_GDR
            cs.updateReconMotIPM(area);
    #else
    
            cs.picture->getRecoBuf( area ).copyFrom( cs.    getRecoBuf( area ) );
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #endif
    
    
            for( uint32_t j = 0; j < saveCS.tus.size(); j++ )
            {
              orgTUs[ j ]->copyComponentFrom( *saveCS.tus[ j ], area.compID );
            }
          }
        }
    
        pu.intraDir[1] = uiBestMode;
        cs.dist        = uiBestDist;
    
        cu.bdpcmModeChroma = bestBDPCMMode;
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
        pu.decoderDerivedCcpMode = decoderDerivedCcpModeBest;
        pu.ddNonLocalCCPFusion = bestDdNonLocalMergeFusion;
    #endif
    
    #if JVET_Z0050_CCLM_SLOPE
        pu.cclmOffsets     = bestCclmOffsets;
    
    Jani Lainema's avatar
    Jani Lainema committed
    #if JVET_AA0057_CCCM
        pu.cccmFlag        = cccmModeBest;
    
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING 
        pu.cccmNoSubFlag   = cccmNoSubBest;
    #endif
    
    #if JVET_AC0054_GLCCCM
        pu.glCccmFlag = glCccmBest;
    #endif
    
    #if JVET_AE0100_BVGCCCM
        pu.bvgCccmFlag = bvgCccmBest;
    #endif
    
    #if JVET_AD0202_CCCM_MDF
        pu.cccmMultiFilterIdx = cccmMultiFilterIdxBest;
    #endif
    
    Jani Lainema's avatar
    Jani Lainema committed
    #endif
    
    #if JVET_Z0050_DIMD_CHROMA_FUSION
        pu.isChromaFusion = isChromaFusion;
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
    #endif
    #if JVET_AA0126_GLM
        pu.glmIdc          = bestGlmIdc;
    
    #endif
    #if JVET_AD0188_CCP_MERGE
        pu.idxNonLocalCCP  = bestNonAdjCCCM;
        pu.curCand         = ccpModelBest;
    
    #endif
    #if JVET_AD0120_LBCCP
        pu.ccInsideFilter = bestCCInsideFilter;
    
    #endif
    #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
        pu.ccpMergeFusionFlag = bestCcpMergeFusionFlag;
        pu.ccpMergeFusionType = bestCcpMergeFusionType;
    
      }
    
      //----- restore context models -----
      m_CABACEstimator->getCtx() = ctxStart;
    
      if( lumaUsesISP && bestCostSoFar >= maxCostAllowed )
      {
        cu.ispMode = 0;
      }
    
    #if JVET_Z0050_CCLM_SLOPE
    void IntraSearch::xFindBestCclmDeltaSlopeSATD(PredictionUnit &pu, ComponentID compID, int cclmModel, int &deltaBest, int64_t &sadBest )
    {
      CclmModel cclmModelStored;
      CodingStructure& cs = *(pu.cs);
      CompArea       area = compID == COMPONENT_Cb ? pu.Cb() : pu.Cr();
      PelBuf       orgBuf = cs.getOrgBuf(area);
      PelBuf      predBuf = cs.getPredBuf(area);
      int       maxOffset = 4;
      int            mode = pu.intraDir[1];
      bool createNewModel = true;
    
      DistParam distParamSad;
      DistParam distParamSatd;
    
      m_pcRdCost->setDistParam(distParamSad,  orgBuf, predBuf, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), compID, false);
      m_pcRdCost->setDistParam(distParamSatd, orgBuf, predBuf, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), compID, true);
      
      distParamSad.applyWeight  = false;
      distParamSatd.applyWeight = false;
      
      sadBest = -1;
    
      // Search positive offsets
      for ( int offset = 0; offset <= maxOffset; offset++)
      {
        pu.cclmOffsets.setOffset(compID, cclmModel, offset);
        
        predIntraChromaLM( compID, predBuf, pu, area, mode, createNewModel, &cclmModelStored );
        
        createNewModel  = false; // Need to calculate the base model just once
        int64_t sad     = distParamSad.distFunc(distParamSad) * 2;
        int64_t satd    = distParamSatd.distFunc(distParamSatd);
        int64_t sadThis = std::min(sad, satd);
    
        if ( sadBest == -1 || sadThis < sadBest )
        {
          sadBest   = sadThis;
          deltaBest = offset;
        }
        else
        {
          break;
        }
      }
      
      // Search negative offsets only if positives didn't help
      if ( deltaBest == 0 )
      {
        for ( int offset = -1; offset >= -maxOffset; offset--)
        {
          pu.cclmOffsets.setOffset(compID, cclmModel, offset);
    
          predIntraChromaLM( compID, predBuf, pu, area, mode, createNewModel, &cclmModelStored );
          
          int64_t sad     = distParamSad.distFunc(distParamSad) * 2;
          int64_t satd    = distParamSatd.distFunc(distParamSatd);
          int64_t sadThis = std::min(sad, satd);
    
          if ( sadThis < sadBest )
          {
            sadBest   = sadThis;
            deltaBest = offset;
          }
          else
          {
            break;
          }
        }
      }
    }
    #endif
    
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
    #if JVET_AA0126_GLM
    void IntraSearch::xFindBestGlmIdcSATD(PredictionUnit &pu, ComponentID compID, int &idcBest, int64_t &sadBest )
    {
      CodingStructure& cs = *(pu.cs);
      CompArea       area = compID == COMPONENT_Cb ? pu.Cb() : pu.Cr();
      PelBuf       orgBuf = cs.getOrgBuf(area);
      PelBuf      predBuf = cs.getPredBuf(area);
    
    #if JVET_AB0092_GLM_WITH_LUMA
      int          maxIdc = NUM_GLM_PATTERN * NUM_GLM_WEIGHT;
    #else
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
      int          maxIdc = NUM_GLM_IDC - 1;
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
      int            mode = pu.intraDir[1];
    
      DistParam distParamSad;
      DistParam distParamSatd;
    
      m_pcRdCost->setDistParam(distParamSad,  orgBuf, predBuf, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), compID, false);
      m_pcRdCost->setDistParam(distParamSatd, orgBuf, predBuf, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), compID, true);
      
      distParamSad.applyWeight  = false;
      distParamSatd.applyWeight = false;
      
      sadBest = -1;
    
    
    #if JVET_AB0092_GLM_WITH_LUMA
      CompArea       areacr = pu.Cr();
      PelBuf       orgBufcr = cs.getOrgBuf(areacr);
      PelBuf      predBufcr = cs.getPredBuf(areacr);
    
      DistParam distParamSadcr;
      DistParam distParamSatdcr;
    
      m_pcRdCost->setDistParam(distParamSadcr, orgBufcr, predBufcr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, false);
      m_pcRdCost->setDistParam(distParamSatdcr, orgBufcr, predBufcr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, true);
    
      distParamSadcr.applyWeight = false;
      distParamSatdcr.applyWeight = false;
    #endif
    
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
      // Search positive idcs
      for ( int idc = 0; idc <= maxIdc; idc++ )
      {
        pu.glmIdc.setIdc(compID, 0, idc);
        pu.glmIdc.setIdc(compID, 1, idc);
    
        predIntraChromaLM( compID, predBuf, pu, area, mode );
        
        int64_t sad     = distParamSad.distFunc(distParamSad) * 2;
        int64_t satd    = distParamSatd.distFunc(distParamSatd);
        int64_t sadThis = std::min(sad, satd);
    
    
    #if JVET_AB0092_GLM_WITH_LUMA
        pu.glmIdc.setIdc(COMPONENT_Cr, 0, idc);
        pu.glmIdc.setIdc(COMPONENT_Cr, 1, idc);
    
        predIntraChromaLM(COMPONENT_Cr, predBufcr, pu, areacr, mode);
    
        int64_t sadcr = distParamSadcr.distFunc(distParamSadcr) * 2;
        int64_t satdcr = distParamSatdcr.distFunc(distParamSatdcr);
        int64_t sadThiscr = std::min(sadcr, satdcr);
        sadThis += sadThiscr;
    #endif
    
    Che-Wei Kuo's avatar
    Che-Wei Kuo committed
        if ( sadBest == -1 || sadThis < sadBest )
        {
          sadBest   = sadThis;
          idcBest   = idc;
        }
      }
    }
    #endif
    
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
    void IntraSearch::saveCuAreaCostInSCIPU( Area area, double cost )
    {
      if( m_numCuInSCIPU < NUM_INTER_CU_INFO_SAVE )
      {
        m_cuAreaInSCIPU[m_numCuInSCIPU] = area;
        m_cuCostInSCIPU[m_numCuInSCIPU] = cost;
        m_numCuInSCIPU++;
      }
    }
    
    void IntraSearch::initCuAreaCostInSCIPU()
    {
      for( int i = 0; i < NUM_INTER_CU_INFO_SAVE; i++ )
      {
        m_cuAreaInSCIPU[i] = Area();
        m_cuCostInSCIPU[i] = 0;
      }
      m_numCuInSCIPU = 0;
    }
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
    void IntraSearch::PLTSearch(CodingStructure &cs, Partitioner& partitioner, ComponentID compBegin, uint32_t numComp)
    
      CodingUnit    &cu = *cs.getCU(partitioner.chType);
    
      TransformUnit &tu = *cs.getTU(partitioner.chType);
    
      uint32_t height = cu.block(compBegin).height;
      uint32_t width = cu.block(compBegin).width;
    
      if (m_pcEncCfg->getLmcs() && (cs.slice->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag()))
    
      {
        cs.getPredBuf().copyFrom(cs.getOrgBuf());
        cs.getPredBuf().Y().rspSignal(m_pcReshape->getFwdLUT());
      }
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
      if( cu.isLocalSepTree() )
    
        cs.prevPLT.curPLTSize[compBegin] = cs.prevPLT.curPLTSize[COMPONENT_Y];
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
      cu.lastPLTSize[compBegin] = cs.prevPLT.curPLTSize[compBegin];
      //derive palette
      derivePLTLossy(cs, partitioner, compBegin, numComp);
      reorderPLT(cs, partitioner, compBegin, numComp);
    
      bool idxExist[MAXPLTSIZE + 1] = { false };
    
      preCalcPLTIndexRD(cs, partitioner, compBegin, numComp); // Pre-calculate distortions for each pixel
    
      double rdCost = MAX_DOUBLE;
    
      deriveIndexMap(cs, partitioner, compBegin, numComp, PLT_SCAN_HORTRAV, rdCost, idxExist); // Optimize palette index map (horizontal scan)
    
      if ((cu.curPLTSize[compBegin] + cu.useEscape[compBegin]) > 1)
      {
    
        deriveIndexMap(cs, partitioner, compBegin, numComp, PLT_SCAN_VERTRAV, rdCost, idxExist); // Optimize palette index map (vertical scan)
      }
      // Remove unused palette entries
      uint8_t newPLTSize = 0;
      int idxMapping[MAXPLTSIZE + 1];
      memset(idxMapping, -1, sizeof(int) * (MAXPLTSIZE + 1));
      for (int i = 0; i < cu.curPLTSize[compBegin]; i++)
      {
        if (idxExist[i])
        {
          idxMapping[i] = newPLTSize;
          newPLTSize++;
        }
    
      idxMapping[cu.curPLTSize[compBegin]] = cu.useEscape[compBegin]? newPLTSize: -1;
      if (newPLTSize != cu.curPLTSize[compBegin]) // there exist unused palette entries
      { // update palette table and reuseflag
        Pel curPLTtmp[MAX_NUM_COMPONENT][MAXPLTSIZE];
        int reuseFlagIdx = 0, curPLTtmpIdx = 0, reuseEntrySize = 0;
        memset(cu.reuseflag[compBegin], false, sizeof(bool) * MAXPLTPREDSIZE);
    
        int compBeginTmp = compBegin;
        int numCompTmp   = numComp;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
        if( cu.isLocalSepTree() )
    
          memset(cu.reuseflag[COMPONENT_Y], false, sizeof(bool) * MAXPLTPREDSIZE);
    
          compBeginTmp = COMPONENT_Y;
          numCompTmp   = (cu.chromaFormat != CHROMA_400) ? 3 : 1;
        }
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
        for (int curIdx = 0; curIdx < cu.curPLTSize[compBegin]; curIdx++)
        {
          if (idxExist[curIdx])
          {
    
            for (int comp = compBeginTmp; comp < (compBeginTmp + numCompTmp); comp++)
    
              curPLTtmp[comp][curPLTtmpIdx] = cu.curPLT[comp][curIdx];
    
            // Update reuse flags
            if (curIdx < cu.reusePLTSize[compBegin])
            {
              bool match = false;
              for (; reuseFlagIdx < cs.prevPLT.curPLTSize[compBegin]; reuseFlagIdx++)
              {
                bool matchTmp = true;
                for (int comp = compBegin; comp < (compBegin + numComp); comp++)
                {
                  matchTmp = matchTmp && (curPLTtmp[comp][curPLTtmpIdx] == cs.prevPLT.curPLT[comp][reuseFlagIdx]);
                }
                if (matchTmp)
                {
                  match = true;
                  break;
                }
              }
              if (match)
              {
                cu.reuseflag[compBegin][reuseFlagIdx] = true;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
                if( cu.isLocalSepTree() )
    
                  cu.reuseflag[COMPONENT_Y][reuseFlagIdx] = true;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
                reuseEntrySize++;
              }
            }
            curPLTtmpIdx++;
          }
        }
        cu.reusePLTSize[compBegin] = reuseEntrySize;
        // update palette table
        cu.curPLTSize[compBegin] = newPLTSize;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
        if( cu.isLocalSepTree() )
    
          cu.curPLTSize[COMPONENT_Y] = newPLTSize;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
        for (int comp = compBeginTmp; comp < (compBeginTmp + numCompTmp); comp++)
    
          memcpy( cu.curPLT[comp], curPLTtmp[comp], sizeof(Pel)*cu.curPLTSize[compBegin]);
    
    Yung-Hsuan Chao (Jessie)'s avatar
    Yung-Hsuan Chao (Jessie) committed
      cu.useRotation[compBegin] = m_bestScanRotationMode;
    
      int indexMaxSize = cu.useEscape[compBegin] ? (cu.curPLTSize[compBegin] + 1) : cu.curPLTSize[compBegin];
      if (indexMaxSize <= 1)
      {
        cu.useRotation[compBegin] = false;
      }
    
      //reconstruct pixel
      PelBuf    curPLTIdx = tu.getcurPLTIdx(compBegin);
    
      for (uint32_t y = 0; y < height; y++)
    
        for (uint32_t x = 0; x < width; x++)
    
          curPLTIdx.at(x, y) = idxMapping[curPLTIdx.at(x, y)];
    
          if (curPLTIdx.at(x, y) == cu.curPLTSize[compBegin])
    
            calcPixelPred(cs, partitioner, y, x, compBegin, numComp);
    
          }
          else
          {
            for (uint32_t compID = compBegin; compID < (compBegin + numComp); compID++)
            {
    
              CompArea area = cu.blocks[compID];
              PelBuf   recBuf = cs.getRecoBuf(area);
    
              uint32_t scaleX = getComponentScaleX((ComponentID)COMPONENT_Cb, cs.sps->getChromaFormatIdc());
              uint32_t scaleY = getComponentScaleY((ComponentID)COMPONENT_Cb, cs.sps->getChromaFormatIdc());
              if (compBegin != COMPONENT_Y || compID == COMPONENT_Y)
              {
    
                recBuf.at(x, y) = cu.curPLT[compID][curPLTIdx.at(x, y)];
    
              else if (compBegin == COMPONENT_Y && compID != COMPONENT_Y && y % (1 << scaleY) == 0 && x % (1 << scaleX) == 0)
    
                recBuf.at(x >> scaleX, y >> scaleY) = cu.curPLT[compID][curPLTIdx.at(x, y)];
    
              }
            }
          }
        }
      }
    
      cs.getPredBuf().fill(0);
      cs.getResiBuf().fill(0);
      cs.getOrgResiBuf().fill(0);
    
      cs.fracBits = MAX_UINT;
      cs.cost = MAX_DOUBLE;
      Distortion distortion = 0;
      for (uint32_t comp = compBegin; comp < (compBegin + numComp); comp++)
      {
        const ComponentID compID = ComponentID(comp);
        CPelBuf reco = cs.getRecoBuf(compID);
        CPelBuf org = cs.getOrgBuf(compID);
    
        if (m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled() || (
          m_pcEncCfg->getLmcs() && (cs.slice->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag())))
    
        {
          const CPelBuf orgLuma = cs.getOrgBuf(cs.area.blocks[COMPONENT_Y]);
    
          if (compID == COMPONENT_Y && !(m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled()))
          {
            const CompArea &areaY = cu.Y();
    
            CompArea tmpArea1(COMPONENT_Y, areaY.chromaFormat, Position(0, 0), areaY.size());
            PelBuf   tmpRecLuma = m_tmpStorageLCU.getBuf(tmpArea1);
    
    Vadim Seregin's avatar
    Vadim Seregin committed
            tmpRecLuma.rspSignal( reco, m_pcReshape->getInvLUT() );
    
            distortion += m_pcRdCost->getDistPart(org, tmpRecLuma, cs.sps->getBitDepth(toChannelType(compID)), compID, DF_SSE_WTD, &orgLuma);
          }
          else
          {
            distortion += m_pcRdCost->getDistPart(org, reco, cs.sps->getBitDepth(toChannelType(compID)), compID, DF_SSE_WTD, &orgLuma);
          }
        }
        else
    
          distortion += m_pcRdCost->getDistPart(org, reco, cs.sps->getBitDepth(toChannelType(compID)), compID, DF_SSE);
    
      cs.dist += distortion;
      const CompArea &area = cu.blocks[compBegin];
      cs.setDecomp(area);
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #if JVET_Z0118_GDR
      cs.updateReconMotIPM(area);
    #else
    
      cs.picture->getRecoBuf(area).copyFrom(cs.getRecoBuf(area));
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #endif
    
    void IntraSearch::calcPixelPredRD(CodingStructure& cs, Partitioner& partitioner, Pel* orgBuf, Pel* paPixelValue, Pel* paRecoValue, ComponentID compBegin, uint32_t numComp)
    {
      CodingUnit &cu = *cs.getCU(partitioner.chType);
      TransformUnit &tu = *cs.getTU(partitioner.chType);
    
      int qp[3];
      int qpRem[3];
      int qpPer[3];
      int quantiserScale[3];
      int quantiserRightShift[3];
      int rightShiftOffset[3];
      int invquantiserRightShift[3];
      int add[3];
      for (uint32_t ch = compBegin; ch < (compBegin + numComp); ch++)
      {
        QpParam cQP(tu, ComponentID(ch));
        qp[ch] = cQP.Qp(true);
        qpRem[ch] = qp[ch] % 6;
        qpPer[ch] = qp[ch] / 6;
        quantiserScale[ch] = g_quantScales[0][qpRem[ch]];
        quantiserRightShift[ch] = QUANT_SHIFT + qpPer[ch];
        rightShiftOffset[ch] = 1 << (quantiserRightShift[ch] - 1);
        invquantiserRightShift[ch] = IQUANT_SHIFT;
        add[ch] = 1 << (invquantiserRightShift[ch] - 1);
      }
    
      for (uint32_t ch = compBegin; ch < (compBegin + numComp); ch++)
      {
        const int  channelBitDepth = cu.cs->sps->getBitDepth(toChannelType((ComponentID)ch));
        paPixelValue[ch] = Pel(std::max<int>(0, ((orgBuf[ch] * quantiserScale[ch] + rightShiftOffset[ch]) >> quantiserRightShift[ch])));
        assert(paPixelValue[ch] < (1 << (channelBitDepth + 1)));
        paRecoValue[ch] = (((paPixelValue[ch] * g_invQuantScales[0][qpRem[ch]]) << qpPer[ch]) + add[ch]) >> invquantiserRightShift[ch];
        paRecoValue[ch] = Pel(ClipBD<int>(paRecoValue[ch], channelBitDepth));//to be checked
      }
    }
    
    void IntraSearch::preCalcPLTIndexRD(CodingStructure& cs, Partitioner& partitioner, ComponentID compBegin, uint32_t numComp)
    {
      CodingUnit &cu = *cs.getCU(partitioner.chType);
      uint32_t height = cu.block(compBegin).height;
      uint32_t width = cu.block(compBegin).width;
    
      bool lossless = (m_pcEncCfg->getCostMode() == COST_LOSSLESS_CODING && cs.slice->isLossless());
    
    
      CPelBuf   orgBuf[3];
      for (int comp = compBegin; comp < (compBegin + numComp); comp++)
      {
        CompArea  area = cu.blocks[comp];
    
        if (m_pcEncCfg->getLmcs() && (cs.slice->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag()))
    
        {
          orgBuf[comp] = cs.getPredBuf(area);
        }
        else
        {
          orgBuf[comp] = cs.getOrgBuf(area);
        }
      }
    
      int rasPos;
      uint32_t scaleX = getComponentScaleX(COMPONENT_Cb, cs.sps->getChromaFormatIdc());
      uint32_t scaleY = getComponentScaleY(COMPONENT_Cb, cs.sps->getChromaFormatIdc());
      for (uint32_t y = 0; y < height; y++)
      {
        for (uint32_t x = 0; x < width; x++)
        {
          rasPos = y * width + x;;
          // chroma discard
          bool discardChroma = (compBegin == COMPONENT_Y) && (y&scaleY || x&scaleX);
          Pel curPel[3];
          for (int comp = compBegin; comp < (compBegin + numComp); comp++)
          {
            uint32_t pX1 = (comp > 0 && compBegin == COMPONENT_Y) ? (x >> scaleX) : x;
            uint32_t pY1 = (comp > 0 && compBegin == COMPONENT_Y) ? (y >> scaleY) : y;
            curPel[comp] = orgBuf[comp].at(pX1, pY1);
          }
    
          uint8_t  pltIdx = 0;
          double minError = MAX_DOUBLE;
          uint8_t  bestIdx = 0;
    
          for (uint8_t z = 0; z < cu.curPLTSize[compBegin]; z++)
          {
            m_indexError[z][rasPos] = minError;
          }
    
          while (pltIdx < cu.curPLTSize[compBegin])
          {
            uint64_t sqrtError = 0;
    
            if (lossless)
            {
              for (int comp = compBegin; comp < (discardChroma ? 1 : (compBegin + numComp)); comp++)
              {
                sqrtError += int64_t(abs(curPel[comp] - cu.curPLT[comp][pltIdx]));
              }
              if (sqrtError == 0)
              {
                m_indexError[pltIdx][rasPos] = (double) sqrtError;
                minError                     = (double) sqrtError;
                bestIdx                      = pltIdx;
                break;
              }
            }
            else
            {
    
              for (int comp = compBegin; comp < (discardChroma ? 1 : (compBegin + numComp)); comp++)
    
                int64_t tmpErr = int64_t(curPel[comp] - cu.curPLT[comp][pltIdx]);
                if (isChroma((ComponentID) comp))
                {
                  sqrtError += uint64_t(tmpErr * tmpErr * ENC_CHROMA_WEIGHTING);
                }
                else
                {
                  sqrtError += tmpErr * tmpErr;
                }
    
              m_indexError[pltIdx][rasPos] = (double) sqrtError;
              if (sqrtError < minError)
    
                minError = (double) sqrtError;
                bestIdx  = pltIdx;
    
              }
            }
            pltIdx++;
          }
    
          Pel paPixelValue[3], paRecoValue[3];
    
            calcPixelPredRD(cs, partitioner, curPel, paPixelValue, paRecoValue, compBegin, numComp);
    
          uint64_t error = 0, rate = 0;
          for (int comp = compBegin; comp < (discardChroma ? 1 : (compBegin + numComp)); comp++)
          {
    
            if (lossless)
            {
              rate += m_escapeNumBins[curPel[comp]];
            }
            else
            {
    
              int64_t tmpErr = int64_t(curPel[comp] - paRecoValue[comp]);
              if (isChroma((ComponentID) comp))
              {
                error += uint64_t(tmpErr * tmpErr * ENC_CHROMA_WEIGHTING);
              }
              else
              {
                error += tmpErr * tmpErr;
              }
              rate += m_escapeNumBins[paPixelValue[comp]];   // encode quantized escape color
    
          }
          double rdCost = (double)error + m_pcRdCost->getLambda()*(double)rate;
          m_indexError[cu.curPLTSize[compBegin]][rasPos] = rdCost;
    
          if (rdCost < minError)
    
          {
            minError = rdCost;
            bestIdx = (uint8_t)cu.curPLTSize[compBegin];
          }
          m_minErrorIndexMap[rasPos] = bestIdx; // save the optimal index of the current pixel
        }
      }
    }
    
    
    void IntraSearch::deriveIndexMap(CodingStructure& cs, Partitioner& partitioner, ComponentID compBegin, uint32_t numComp, PLTScanMode pltScanMode, double& dMinCost, bool* idxExist)
    
    {
      CodingUnit    &cu = *cs.getCU(partitioner.chType);