Skip to content
Snippets Groups Projects
EncCu.cpp 172 KiB
Newer Older
  • Learn to ignore specific revisions
  •   }
    
      if( m_pcEncCfg->getUseGBiFast() )
      {
        if( isEqualUni == true && m_pcEncCfg->getIntraPeriod() == -1 )
        {
          break;
        }
      }
      if( g_GbiSearchOrder[gbiLoopIdx] == GBI_DEFAULT && xIsGBiSkip(cu) && m_pcEncCfg->getUseGBiFast() )
      {
        break;
      }
     }  // for( UChar gbiLoopIdx = 0; gbiLoopIdx < gbiLoopNum; gbiLoopIdx++ )
    
    Nan Hu's avatar
    Nan Hu committed
    #if JVET_M0428_ENC_DB_OPT
     if ( m_bestModeUpdated && bestCS->cost != MAX_DOUBLE )
       xCalDebCost( *bestCS, partitioner );
    #endif
    
    }
    
    
    
    
    
    bool EncCu::xCheckRDCostInterIMV( CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner, const EncTestMode& encTestMode )
    {
      int iIMV = int( ( encTestMode.opts & ETO_IMV ) >> ETO_IMV_SHIFT );
    
      m_pcInterSearch->setAffineModeSelected(false);
    
      // Only int-Pel, 4-Pel and fast 4-Pel allowed
      CHECK( iIMV != 1 && iIMV != 2 && iIMV != 3, "Unsupported IMV Mode" );
      // Fast 4-Pel Mode
    
    
    Nan Hu's avatar
    Nan Hu committed
    #if JVET_M0428_ENC_DB_OPT
      m_bestModeUpdated = tempCS->useDbCost = bestCS->useDbCost = false;
    #endif
    
    
      EncTestMode encTestModeBase = encTestMode;                                        // copy for clearing non-IMV options
      encTestModeBase.opts        = EncTestModeOpts( encTestModeBase.opts & ETO_IMV );  // clear non-IMV options (is that intended?)
    
      tempCS->initStructData( encTestMode.qp, encTestMode.lossless );
    
      CodingStructure* pcCUInfo2Reuse = nullptr;
    
    
      m_pcInterSearch->resetBufferedUniMotions();
      int gbiLoopNum = (tempCS->slice->isInterB() ? GBI_NUM : 1);
      gbiLoopNum = (pcCUInfo2Reuse != NULL ? 1 : gbiLoopNum);
    
      gbiLoopNum = (tempCS->slice->getSPS()->getUseGBi() ? gbiLoopNum : 1);
    
    
      if( tempCS->area.lwidth() * tempCS->area.lheight() < GBI_SIZE_CONSTRAINT )
      {
        gbiLoopNum = 1;
      }
    
    
    #if JVET_M0246_AFFINE_AMVR
      bool validMode = false;
    #endif
    
      double curBestCost = bestCS->cost;
      double equGBiCost = MAX_DOUBLE;
    
      for( int gbiLoopIdx = 0; gbiLoopIdx < gbiLoopNum; gbiLoopIdx++ )
      {
        if( m_pcEncCfg->getUseGBiFast() )
        {
          auto blkCache = dynamic_cast< CacheBlkInfoCtrl* >(m_modeCtrl);
    
          if( blkCache )
          {
            bool isBestInter = blkCache->getInter(bestCS->area);
            uint8_t bestGBiIdx = blkCache->getGbiIdx(bestCS->area);
    
            if( isBestInter && g_GbiSearchOrder[gbiLoopIdx] != GBI_DEFAULT && g_GbiSearchOrder[gbiLoopIdx] != bestGBiIdx )
            {
              continue;
            }
          }
        }
    
        if( !tempCS->slice->getCheckLDC() )
        {
          if( gbiLoopIdx != 0 && gbiLoopIdx != 3 && gbiLoopIdx != 4 )
          {
            continue;
          }
        }
    
        if( m_pcEncCfg->getUseGBiFast() && tempCS->slice->getCheckLDC() && g_GbiSearchOrder[gbiLoopIdx] != GBI_DEFAULT
          && (m_bestGbiIdx[0] >= 0 && g_GbiSearchOrder[gbiLoopIdx] != m_bestGbiIdx[0])
          && (m_bestGbiIdx[1] >= 0 && g_GbiSearchOrder[gbiLoopIdx] != m_bestGbiIdx[1]))
        {
          continue;
        }
    
      CodingUnit &cu = ( pcCUInfo2Reuse != nullptr ) ? *tempCS->getCU( partitioner.chType ) : tempCS->addCU( tempCS->area, partitioner.chType );
    
      if( pcCUInfo2Reuse == nullptr )
      {
        partitioner.setCUData( cu );
        cu.slice            = tempCS->slice;
    #if HEVC_TILES_WPP
        cu.tileIdx          = tempCS->picture->tileMap->getTileIdxMap( tempCS->area.lumaPos() );
    #endif
        cu.skip             = false;
    
      //cu.affine
        cu.predMode         = MODE_INTER;
        cu.transQuantBypass = encTestMode.lossless;
        cu.chromaQpAdj      = cu.transQuantBypass ? 0 : m_cuChromaQpOffsetIdxPlus1;
        cu.qp               = encTestMode.qp;
    
        CU::addPUs( cu );
      }
      else
      {
        CHECK( cu.skip,                                "Mismatch" );
        CHECK( cu.qtDepth  != partitioner.currQtDepth, "Mismatch" );
        CHECK( cu.btDepth  != partitioner.currBtDepth, "Mismatch" );
        CHECK( cu.mtDepth  != partitioner.currMtDepth, "Mismatch" );
        CHECK( cu.depth    != partitioner.currDepth,   "Mismatch" );
      }
    
      cu.imv      = iIMV > 1 ? 2 : 1;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
      bool testGbi;
      uint8_t gbiIdx;
    
    #if JVET_M0246_AFFINE_AMVR
      bool affineAmvrEanbledFlag = cu.slice->getSPS()->getAffineAmvrEnabledFlag();
    #endif
    
      if( pcCUInfo2Reuse != nullptr )
      {
        // reuse the motion info from pcCUInfo2Reuse
        CU::resetMVDandMV2Int( cu, m_pcInterSearch );
    
    
        CHECK(cu.GBiIdx < 0 || cu.GBiIdx >= GBI_NUM, "cu.GBiIdx < 0 || cu.GBiIdx >= GBI_NUM");
        gbiIdx = CU::getValidGbiIdx(cu);
        testGbi = (gbiIdx != GBI_DEFAULT);
    
    #if JVET_M0246_AFFINE_AMVR
        if ( !CU::hasSubCUNonZeroMVd( cu ) && !CU::hasSubCUNonZeroAffineMVd( cu ) )
    #else
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
          if (m_modeCtrl->useModeResult(encTestModeBase, tempCS, partitioner))
          {
            std::swap(tempCS, bestCS);
            // store temp best CI for next CU coding
            m_CurrCtx->best = m_CABACEstimator->getCtx();
          }
    
    #if JVET_M0246_AFFINE_AMVR
          if ( affineAmvrEanbledFlag )
          {
            tempCS->initStructData( encTestMode.qp, encTestMode.lossless );
            continue;
          }
          else
          {
            return false;
          }
    #else
    
        cu.GBiIdx = g_GbiSearchOrder[gbiLoopIdx];
        gbiIdx = cu.GBiIdx;
        testGbi = (gbiIdx != GBI_DEFAULT);
    
    #if JVET_M0246_AFFINE_AMVR
        cu.firstPU->interDir = 10;
    #endif
    
    
        m_pcInterSearch->predInterSearch( cu, partitioner );
    
    #if JVET_M0246_AFFINE_AMVR
        if ( cu.firstPU->interDir <= 3 )
        {
          gbiIdx = CU::getValidGbiIdx(cu);
        }
        else
        {
          return false;
        }
    #else
    
        gbiIdx = CU::getValidGbiIdx(cu);
    
      if( testGbi && gbiIdx == GBI_DEFAULT ) // Enabled GBi but the search results is uni.
      {
        tempCS->initStructData(encTestMode.qp, encTestMode.lossless);
        continue;
      }
      CHECK(!(testGbi || (!testGbi && gbiIdx == GBI_DEFAULT)), " !( bTestGbi || (!bTestGbi && gbiIdx == GBI_DEFAULT ) )");
    
      bool isEqualUni = false;
      if( m_pcEncCfg->getUseGBiFast() )
      {
        if( cu.firstPU->interDir != 3 && testGbi == 0 )
        {
          isEqualUni = true;
        }
      }
    
    #if JVET_M0246_AFFINE_AMVR
      if ( !CU::hasSubCUNonZeroMVd( cu ) && !CU::hasSubCUNonZeroAffineMVd( cu ) )
    #else
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
        if (m_modeCtrl->useModeResult(encTestModeBase, tempCS, partitioner))
        {
          std::swap(tempCS, bestCS);
          // store temp best CI for next CU coding
          m_CurrCtx->best = m_CABACEstimator->getCtx();
        }
    
    #if JVET_M0246_AFFINE_AMVR
        if ( affineAmvrEanbledFlag )
        {
          tempCS->initStructData( encTestMode.qp, encTestMode.lossless );
          continue;
        }
        else
        {
          return false;
        }
    #else
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if JVET_M0464_UNI_MTS
      xEncodeInterResidual( tempCS, bestCS, partitioner, encTestModeBase, 0
                            , NULL
                            , 0
                            , &equGBiCost
    #else
    
      xEncodeInterResidual( tempCS, bestCS, partitioner, encTestModeBase, 0
        , NULL
        , true
        , 0
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
      tempCS->initStructData(encTestMode.qp, encTestMode.lossless);
    
      double skipTH = MAX_DOUBLE;
      skipTH = (m_pcEncCfg->getUseGBiFast() ? 1.05 : MAX_DOUBLE);
      if( equGBiCost > curBestCost * skipTH )
      {
        break;
      }
    
      if( m_pcEncCfg->getUseGBiFast() )
      {
        if( isEqualUni == true && m_pcEncCfg->getIntraPeriod() == -1 )
        {
          break;
        }
      }
      if( g_GbiSearchOrder[gbiLoopIdx] == GBI_DEFAULT && xIsGBiSkip(cu) && m_pcEncCfg->getUseGBiFast() )
      {
        break;
      }
    
    #if JVET_M0246_AFFINE_AMVR
      validMode = true;
    #endif
    
     } // for( UChar gbiLoopIdx = 0; gbiLoopIdx < gbiLoopNum; gbiLoopIdx++ )
    
    
    Nan Hu's avatar
    Nan Hu committed
    #if JVET_M0428_ENC_DB_OPT
     if ( m_bestModeUpdated && bestCS->cost != MAX_DOUBLE )
       xCalDebCost( *bestCS, partitioner );
    #endif
    
    
    #if JVET_M0246_AFFINE_AMVR
      return tempCS->slice->getSPS()->getAffineAmvrEnabledFlag() ? validMode : true;
    #else
    
    Nan Hu's avatar
    Nan Hu committed
    #if JVET_M0428_ENC_DB_OPT
    void EncCu::xCalDebCost(CodingStructure &cs, Partitioner &partitioner, bool calDist)
    {
      if (cs.cost == MAX_DOUBLE)
        cs.costDbOffset = 0;
    
      if (cs.slice->getDeblockingFilterDisable() || (!m_pcEncCfg->getUseEncDbOpt() && !calDist))
        return;
    
      m_pcLoopFilter->setEnc(true);
      const ChromaFormat format = cs.area.chromaFormat;
      CodingUnit*                cu = cs.getCU(partitioner.chType);
      const Position lumaPos = cu->Y().valid() ? cu->Y().pos() : recalcPosition(format, cu->chType, CHANNEL_TYPE_LUMA, cu->blocks[cu->chType].pos());
      bool topEdgeAvai  = lumaPos.y > 0 && ((lumaPos.y % 8) == 0);
      bool leftEdgeAvai = lumaPos.x > 0 && ((lumaPos.x % 8) == 0);
      bool anyEdgeAvai = topEdgeAvai || leftEdgeAvai;
      cs.costDbOffset = 0;
    
      if (calDist)
      {
        const UnitArea currCsArea = clipArea(CS::getArea(cs, cs.area, partitioner.chType), *cs.picture);
        ComponentID compStr = (CS::isDualITree(cs) && !isLuma(partitioner.chType)) ? COMPONENT_Cb : COMPONENT_Y;
        ComponentID compEnd = (CS::isDualITree(cs) && isLuma(partitioner.chType)) ? COMPONENT_Y : COMPONENT_Cr;
        Distortion finalDistortion = 0;
        for (int comp = compStr; comp <= compEnd; comp++)
        {
          const ComponentID compID = ComponentID(comp);
          CPelBuf org = cs.getOrgBuf(compID);
          CPelBuf reco = cs.getRecoBuf(compID);
          finalDistortion += getDistortionDb(cs, org, reco, compID, currCsArea.block(compID), false);
        }
        //updated distortion
        cs.dist = finalDistortion;
      }
    
      if (anyEdgeAvai && m_pcEncCfg->getUseEncDbOpt())
      {
        ComponentID compStr = (CS::isDualITree(cs) && !isLuma(partitioner.chType)) ? COMPONENT_Cb : COMPONENT_Y;
        ComponentID compEnd = (CS::isDualITree(cs) && isLuma(partitioner.chType)) ? COMPONENT_Y : COMPONENT_Cr;
    
        const UnitArea currCsArea = clipArea(CS::getArea(cs, cs.area, partitioner.chType), *cs.picture);
    
        PelStorage&          picDbBuf = m_pcLoopFilter->getDbEncPicYuvBuffer();
    
        //deblock neighbour pixels
        const Size     lumaSize = cu->Y().valid() ? cu->Y().size() : recalcSize(format, cu->chType, CHANNEL_TYPE_LUMA, cu->blocks[cu->chType].size());
    
    #if  JVET_M0471_LONG_DEBLOCKING_FILTERS
        const int verOffset = lumaPos.y > 7 ? 8 : 4;
        const int horOffset = lumaPos.x > 7 ? 8 : 4;
    #else
        const int verOffset = 4;
        const int horOffset = 4;
    #endif
        const UnitArea areaTop(format, Area(lumaPos.x, lumaPos.y - verOffset, lumaSize.width, verOffset));
        const UnitArea areaLeft(format, Area(lumaPos.x - horOffset, lumaPos.y, horOffset, lumaSize.height));
        for (int compIdx = compStr; compIdx <= compEnd; compIdx++)
        {
          ComponentID compId = (ComponentID)compIdx;
    
          //Copy current CU's reco to Deblock Pic Buffer
          const CompArea&  curCompArea = currCsArea.block(compId);
          picDbBuf.getBuf(curCompArea).copyFrom(cs.getRecoBuf(curCompArea));
    #if JVET_M0427_INLOOP_RESHAPER
          if (cs.slice->getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getSliceReshaperInfo().getUseSliceReshaper() && isLuma(compId))
          {
            picDbBuf.getBuf(curCompArea).rspSignal(m_pcReshape->getInvLUT());
          }
    #endif
    
          //left neighbour
          if (leftEdgeAvai)
          {
            const CompArea&  compArea = areaLeft.block(compId);
            picDbBuf.getBuf(compArea).copyFrom(cs.picture->getRecoBuf(compArea));
    #if JVET_M0427_INLOOP_RESHAPER       
            if (cs.slice->getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getSliceReshaperInfo().getUseSliceReshaper() && isLuma(compId))
            {
              picDbBuf.getBuf(compArea).rspSignal(m_pcReshape->getInvLUT());
            }
    #endif
          }
          //top neighbour
          if (topEdgeAvai)
          {
            const CompArea&  compArea = areaTop.block(compId);
            picDbBuf.getBuf(compArea).copyFrom(cs.picture->getRecoBuf(compArea));
    #if JVET_M0427_INLOOP_RESHAPER
            if (cs.slice->getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getSliceReshaperInfo().getUseSliceReshaper() && isLuma(compId))
            {
              picDbBuf.getBuf(compArea).rspSignal(m_pcReshape->getInvLUT());
            }
    #endif
          }
        }
    
        //deblock
        if (leftEdgeAvai)
        {
          m_pcLoopFilter->xDeblockCU(*cu, EDGE_VER);
        }
    
        if (topEdgeAvai)
        {
          m_pcLoopFilter->xDeblockCU(*cu, EDGE_HOR);
        }
    
        //update current CU SSE
        Distortion distCur = 0;
        for (int compIdx = compStr; compIdx <= compEnd; compIdx++)
        {
          ComponentID compId = (ComponentID)compIdx;
          CPelBuf reco = picDbBuf.getBuf(currCsArea.block(compId));
          CPelBuf org = cs.getOrgBuf(compId);
          distCur += getDistortionDb(cs, org, reco, compId, currCsArea.block(compId), true);
        }
    
        //calculate difference between DB_before_SSE and DB_after_SSE for neighbouring CUs
        Distortion distBeforeDb = 0, distAfterDb = 0;
        for (int compIdx = compStr; compIdx <= compEnd; compIdx++)
        {
          ComponentID compId = (ComponentID)compIdx;
          if (leftEdgeAvai)
          {
            const CompArea&  compArea = areaLeft.block(compId);
            CPelBuf org = cs.picture->getOrigBuf(compArea);
            CPelBuf reco = cs.picture->getRecoBuf(compArea);
            CPelBuf recoDb = picDbBuf.getBuf(compArea);
            distBeforeDb += getDistortionDb(cs, org, reco,   compId, compArea, false);
            distAfterDb  += getDistortionDb(cs, org, recoDb, compId, compArea, true);
          }
          if (topEdgeAvai)
          {
            const CompArea&  compArea = areaTop.block(compId);
            CPelBuf org = cs.picture->getOrigBuf(compArea);
            CPelBuf reco = cs.picture->getRecoBuf(compArea);
            CPelBuf recoDb = picDbBuf.getBuf(compArea);
            distBeforeDb += getDistortionDb(cs, org, reco,   compId, compArea, false);
            distAfterDb  += getDistortionDb(cs, org, recoDb, compId, compArea, true);
          }
        }
    
        //updated cost
        int64_t distTmp = distCur - cs.dist + distAfterDb - distBeforeDb;
        int sign = distTmp < 0 ? -1 : 1;
        distTmp = distTmp < 0 ? -distTmp : distTmp;
        cs.costDbOffset = sign * m_pcRdCost->calcRdCost(0, distTmp);
      }
    
    Nan Hu's avatar
    Nan Hu committed
    
    
    Nan Hu's avatar
    Nan Hu committed
      m_pcLoopFilter->setEnc(false);
    }
    
    Distortion EncCu::getDistortionDb(CodingStructure &cs, CPelBuf org, CPelBuf reco, ComponentID compID, const CompArea& compArea, bool afterDb)
    {
      Distortion dist = 0;
    #if WCG_EXT
      CPelBuf orgLuma = cs.picture->getOrigBuf(compArea);
    #if JVET_M0427_INLOOP_RESHAPER
      if (m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled() || (
        m_pcEncCfg->getReshaper() && (cs.slice->getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getCTUFlag())))
    #else
      if (m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled())
    #endif
      {
    #if JVET_M0427_INLOOP_RESHAPER
        if (compID == COMPONENT_Y && !afterDb)
        {
          CompArea    tmpArea(COMPONENT_Y, cs.area.chromaFormat, Position(0, 0), compArea.size());
          PelBuf tmpRecLuma = m_tmpStorageLCU->getBuf(tmpArea);
          tmpRecLuma.copyFrom(reco);
          tmpRecLuma.rspSignal(m_pcReshape->getInvLUT());
          dist += m_pcRdCost->getDistPart(org, tmpRecLuma, cs.sps->getBitDepth(toChannelType(compID)), compID, DF_SSE_WTD, &orgLuma);
        }
        else
    #endif
        {
          dist += m_pcRdCost->getDistPart(org, reco, cs.sps->getBitDepth(toChannelType(compID)), compID, DF_SSE_WTD, &orgLuma);
        }
      }
    #if JVET_M0427_INLOOP_RESHAPER
      else if (m_pcEncCfg->getReshaper() && cs.slice->getReshapeInfo().getUseSliceReshaper() && cs.slice->isIntra()) //intra slice
      {
        if (compID == COMPONENT_Y && afterDb)
        {
          CompArea    tmpArea(COMPONENT_Y, cs.area.chromaFormat, Position(0, 0), compArea.size());
          PelBuf tmpRecLuma = m_tmpStorageLCU->getBuf(tmpArea);
          tmpRecLuma.copyFrom(reco);
          tmpRecLuma.rspSignal(m_pcReshape->getFwdLUT());
          dist += m_pcRdCost->getDistPart(org, tmpRecLuma, cs.sps->getBitDepth(toChannelType(compID)), compID, DF_SSE);
        }
        else
        {
          dist += m_pcRdCost->getDistPart(org, reco, cs.sps->getBitDepth(toChannelType(compID)), compID, DF_SSE);
        }
      }
    #endif
      else
    #endif
      {
        dist = m_pcRdCost->getDistPart(org, reco, cs.sps->getBitDepth(toChannelType(compID)), compID, DF_SSE);
      }
      return dist;
    }
    #endif
    
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if JVET_M0464_UNI_MTS
    void EncCu::xEncodeInterResidual(   CodingStructure *&tempCS
                                      , CodingStructure *&bestCS
                                      , Partitioner &partitioner
                                      , const EncTestMode& encTestMode
                                      , int residualPass
                                      , CodingStructure* imvCS
                                      , bool* bestHasNonResi
                                      , double* equGBiCost
    #else
    
    void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner, const EncTestMode& encTestMode, int residualPass
      , CodingStructure* imvCS
      , int emtMode
      , bool* bestHasNonResi
    
      , double* equGBiCost
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    {
      if( residualPass == 1 && encTestMode.lossless )
      {
        return;
      }
    
      CodingUnit*            cu        = tempCS->getCU( partitioner.chType );
      double   bestCostInternal        = MAX_DOUBLE;
    
    #if JVET_M0140_SBT
      double           bestCostBegin   = bestCS->cost;
      CodingUnit*      prevBestCU      = bestCS->getCU( partitioner.chType );
      uint8_t          prevBestSbt     = ( prevBestCU == nullptr ) ? 0 : prevBestCU->sbtInfo;
    #endif
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Karsten Suehring's avatar
    Karsten Suehring committed
      const int      maxSizeEMT        = EMT_INTER_MAX_CU_WITH_QTBT;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
      bool              swapped        = false; // avoid unwanted data copy
      bool             reloadCU        = false;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
      const bool considerEmtSecondPass = emtMode && sps.getUseInterEMT() && partitioner.currArea().lwidth() <= maxSizeEMT && partitioner.currArea().lheight() <= maxSizeEMT;
    
    
      int minEMTMode = 0;
      int maxEMTMode = (considerEmtSecondPass?1:0);
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    Xiang Li's avatar
    Xiang Li committed
      // Not allow very big |MVd| to avoid CABAC crash caused by too large MVd. Normally no impact on coding performance.
      const int maxMvd = 1 << 15;
      const PredictionUnit& pu = *cu->firstPU;
      if (!cu->affine)
      {
        if ((pu.refIdx[0] >= 0 && (pu.mvd[0].getAbsHor() >= maxMvd || pu.mvd[0].getAbsVer() >= maxMvd))
          || (pu.refIdx[1] >= 0 && (pu.mvd[1].getAbsHor() >= maxMvd || pu.mvd[1].getAbsVer() >= maxMvd)))
        {
          return;
        }
      }
    
      else
      {
        for (int refList = 0; refList < NUM_REF_PIC_LIST_01; refList++)
        {
          if (pu.refIdx[refList] >= 0)
          {
            for (int ctrlP = 1 + (cu->affineType == AFFINEMODEL_6PARAM); ctrlP >= 0; ctrlP--)
            {
              if (pu.mvdAffi[refList][ctrlP].getAbsHor() >= maxMvd || pu.mvdAffi[refList][ctrlP].getAbsVer() >= maxMvd)
              {
                return;
              }
            }
          }
        }
      }
    
      const bool mtsAllowed = tempCS->sps->getUseInterMTS() && partitioner.currArea().lwidth() <= MTS_INTER_MAX_CU_SIZE && partitioner.currArea().lheight() <= MTS_INTER_MAX_CU_SIZE;
    
      uint8_t sbtAllowed = cu->checkAllowedSbt();
      uint8_t numRDOTried = 0;
      Distortion sbtOffDist = 0;
      bool    sbtOffRootCbf = 0;
      double  sbtOffCost      = MAX_DOUBLE;
      double  currBestCost = MAX_DOUBLE;
      bool    doPreAnalyzeResi = ( sbtAllowed || mtsAllowed ) && residualPass == 0;
    
      m_pcInterSearch->initTuAnalyzer();
      if( doPreAnalyzeResi )
      {
        m_pcInterSearch->calcMinDistSbt( *tempCS, *cu, sbtAllowed );
      }
    
      auto    slsSbt = dynamic_cast<SaveLoadEncInfoSbt*>( m_modeCtrl );
      int     slShift = 4 + std::min( (int)gp_sizeIdxInfo->idxFrom( cu->lwidth() ) + (int)gp_sizeIdxInfo->idxFrom( cu->lheight() ), 9 );
      Distortion curPuSse = m_pcInterSearch->getEstDistSbt( NUMBER_SBT_MODE );
      uint8_t currBestSbt = 0;
      uint8_t currBestTrs = MAX_UCHAR;
      uint8_t histBestSbt = MAX_UCHAR;
      uint8_t histBestTrs = MAX_UCHAR;
      m_pcInterSearch->setHistBestTrs( MAX_UCHAR, MAX_UCHAR );
      if( doPreAnalyzeResi )
      {
        if( m_pcInterSearch->getSkipSbtAll() && !mtsAllowed ) //emt is off
        {
          histBestSbt = 0; //try DCT2
          m_pcInterSearch->setHistBestTrs( histBestSbt, histBestTrs );
        }
        else
        {
          assert( curPuSse != std::numeric_limits<uint64_t>::max() );
          uint16_t compositeSbtTrs = slsSbt->findBestSbt( cu->cs->area, (uint32_t)( curPuSse >> slShift ) );
          histBestSbt = ( compositeSbtTrs >> 0 ) & 0xff;
          histBestTrs = ( compositeSbtTrs >> 8 ) & 0xff;
          if( m_pcInterSearch->getSkipSbtAll() && CU::isSbtMode( histBestSbt ) ) //special case, skip SBT when loading SBT
          {
            histBestSbt = 0; //try DCT2
          }
          m_pcInterSearch->setHistBestTrs( histBestSbt, histBestTrs );
        }
      }
    #endif
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
      if( emtMode == 2 )
      {
        minEMTMode = maxEMTMode = (cu->emtFlag?1:0);
      }
    
      for( int curEmtMode = minEMTMode; curEmtMode <= maxEMTMode; curEmtMode++ )
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
        if( reloadCU )
        {
          if( bestCost == bestCS->cost ) //The first EMT pass didn't become the bestCS, so we clear the TUs generated
          {
            tempCS->clearTUs();
          }
          else if( false == swapped )
          {
            tempCS->initStructData( encTestMode.qp, encTestMode.lossless );
            tempCS->copyStructure( *bestCS, partitioner.chType );
            tempCS->getPredBuf().copyFrom( bestCS->getPredBuf() );
            bestCost = bestCS->cost;
            cu       = tempCS->getCU( partitioner.chType );
            swapped = true;
          }
          else
          {
            tempCS->clearTUs();
            bestCost = bestCS->cost;
            cu       = tempCS->getCU( partitioner.chType );
          }
    
          //we need to restart the distortion for the new tempCS, the bit count and the cost
          tempCS->dist     = 0;
          tempCS->fracBits = 0;
          tempCS->cost     = MAX_DOUBLE;
    
    Nan Hu's avatar
    Nan Hu committed
    #if JVET_M0428_ENC_DB_OPT
          tempCS->costDbOffset = 0;
    #endif
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    #if JVET_M0140_SBT
        cu->sbtInfo = 0;
    #endif
    
    #if JVET_M0140_SBT // skip DCT-2 and EMT if historical best transform mode is SBT
        if( skipResidual || histBestSbt == MAX_UCHAR || !CU::isSbtMode( histBestSbt ) )
        {
    #endif
    
        m_pcInterSearch->encodeResAndCalcRdInterCU( *tempCS, partitioner, skipResidual );
    
    #if JVET_M0140_SBT
        numRDOTried += mtsAllowed ? 2 : 1;
    #endif
    
        xEncodeDontSplit( *tempCS, partitioner );
    
        xCheckDQP( *tempCS, partitioner );
    
    
    #if !JVET_M0140_SBT //harmonize with GBI fast algorithm (move the code to the end of this function)
    
        if( ETM_INTER_ME == encTestMode.type )
        {
          if( equGBiCost != NULL )
          {
            if( tempCS->cost < (*equGBiCost) && cu->GBiIdx == GBI_DEFAULT )
            {
              (*equGBiCost) = tempCS->cost;
            }
          }
          else
          {
            CHECK(equGBiCost == NULL, "equGBiCost == NULL");
          }
          if( tempCS->slice->getCheckLDC() && !cu->imv && cu->GBiIdx != GBI_DEFAULT && tempCS->cost < m_bestGbiCost[1] )
          {
            if( tempCS->cost < m_bestGbiCost[0] )
            {
              m_bestGbiCost[1] = m_bestGbiCost[0];
              m_bestGbiCost[0] = tempCS->cost;
              m_bestGbiIdx[1] = m_bestGbiIdx[0];
              m_bestGbiIdx[0] = cu->GBiIdx;
            }
            else
            {
              m_bestGbiCost[1] = tempCS->cost;
              m_bestGbiIdx[1] = cu->GBiIdx;
            }
          }
        }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
        double emtFirstPassCost = tempCS->cost;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
        if( imvCS && (tempCS->cost < imvCS->cost) )
        {
          if( imvCS->cost != MAX_DOUBLE )
          {
            imvCS->initStructData( encTestMode.qp, encTestMode.lossless );
          }
          imvCS->copyStructure( *tempCS, partitioner.chType );
        }
        if( NULL != bestHasNonResi && (bestCostInternal > tempCS->cost) )
        {
          bestCostInternal = tempCS->cost;
    
          if (!(tempCS->getPU(partitioner.chType)->mhIntraFlag))
    
          if (tempCS->getPU(partitioner.chType)->mhIntraFlag)
    
    Nan Hu's avatar
    Nan Hu committed
    #if JVET_M0428_ENC_DB_OPT
            tempCS->costDbOffset = 0;
    #endif
    
    #if JVET_M0140_SBT
        currBestCost = tempCS->cost;
        sbtOffCost = tempCS->cost;
        sbtOffDist = tempCS->dist;
        sbtOffRootCbf = cu->rootCbf;
        currBestSbt = CU::getSbtInfo( cu->firstTU->mtsIdx > 1 ? SBT_OFF_MTS : SBT_OFF_DCT, 0 );
        currBestTrs = cu->firstTU->mtsIdx;
        if( cu->lwidth() <= MAX_TU_SIZE_FOR_PROFILE && cu->lheight() <= MAX_TU_SIZE_FOR_PROFILE )
        {
          CHECK( tempCS->tus.size() != 1, "tu must be only one" );
        }
    #endif
    
    #if WCG_EXT
        DTRACE_MODE_COST( *tempCS, m_pcRdCost->getLambda( true ) );
    #else
        DTRACE_MODE_COST( *tempCS, m_pcRdCost->getLambda() );
    #endif
        xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );
    
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
        //now we check whether the second pass should be skipped or not
        if( !curEmtMode && maxEMTMode )
        {
          const double thresholdToSkipEmtSecondPass = 1.1; // Skip checking EMT transforms
          const bool bCond1 = !cu->firstTU->cbf[COMPONENT_Y];
    
          const bool bCond3 = emtFirstPassCost > ( bestCost * thresholdToSkipEmtSecondPass );
    
    
          if( m_pcEncCfg->getFastInterEMT() && (bCond1 || bCond3 ) )
    
    #endif
    #if JVET_M0140_SBT // skip DCT-2 and EMT
        }
    #endif
    
    #if JVET_M0140_SBT //RDO for SBT
        uint8_t numSbtRdo = CU::numSbtModeRdo( sbtAllowed );
        //early termination if all SBT modes are not allowed
        //normative
        if( !sbtAllowed || skipResidual )
        {
          numSbtRdo = 0;
        }
        //fast algorithm
        if( ( histBestSbt != MAX_UCHAR && !CU::isSbtMode( histBestSbt ) ) || m_pcInterSearch->getSkipSbtAll() )
        {
          numSbtRdo = 0;
        }
        if( bestCost != MAX_DOUBLE && sbtOffCost != MAX_DOUBLE )
        {
          double th = 1.07;
          if( !( prevBestSbt == 0 || m_sbtCostSave[0] == MAX_DOUBLE ) )
          {
            assert( m_sbtCostSave[1] <= m_sbtCostSave[0] );
            th *= ( m_sbtCostSave[0] / m_sbtCostSave[1] );
          }
          if( sbtOffCost > bestCost * th )
          {
            numSbtRdo = 0;
          }
        }
        if( !sbtOffRootCbf && sbtOffCost != MAX_DOUBLE )
        {
          double th = Clip3( 0.05, 0.55, ( 27 - cu->qp ) * 0.02 + 0.35 );
          if( sbtOffCost < m_pcRdCost->calcRdCost( ( cu->lwidth() * cu->lheight() ) << SCALE_BITS, 0 ) * th )
          {
            numSbtRdo = 0;
          }
        }
    
        if( histBestSbt != MAX_UCHAR && numSbtRdo != 0 )
        {
          numSbtRdo = 1;
          m_pcInterSearch->initSbtRdoOrder( CU::getSbtMode( CU::getSbtIdx( histBestSbt ), CU::getSbtPos( histBestSbt ) ) );
        }
    
        for( int sbtModeIdx = 0; sbtModeIdx < numSbtRdo; sbtModeIdx++ )
        {
          uint8_t sbtMode = m_pcInterSearch->getSbtRdoOrder( sbtModeIdx );
          uint8_t sbtIdx = CU::getSbtIdxFromSbtMode( sbtMode );
          uint8_t sbtPos = CU::getSbtPosFromSbtMode( sbtMode );
    
          //fast algorithm (early skip, save & load)
          if( histBestSbt == MAX_UCHAR )
          {
            uint8_t skipCode = m_pcInterSearch->skipSbtByRDCost( cu->lwidth(), cu->lheight(), cu->mtDepth, sbtIdx, sbtPos, bestCS->cost, sbtOffDist, sbtOffCost, sbtOffRootCbf );
            if( skipCode != MAX_UCHAR )
            {
              continue;
            }
    
            if( sbtModeIdx > 0 )
            {
              uint8_t prevSbtMode = m_pcInterSearch->getSbtRdoOrder( sbtModeIdx - 1 );
              //make sure the prevSbtMode is the same size as the current SBT mode (otherwise the estimated dist may not be comparable)
              if( CU::isSameSbtSize( prevSbtMode, sbtMode ) )
              {
                Distortion currEstDist = m_pcInterSearch->getEstDistSbt( sbtMode );
                Distortion prevEstDist = m_pcInterSearch->getEstDistSbt( prevSbtMode );
                if( currEstDist > prevEstDist * 1.15 )
                {
                  continue;
                }
              }
            }
          }
    
          //init tempCS and TU
          if( bestCost == bestCS->cost ) //The first EMT pass didn't become the bestCS, so we clear the TUs generated
          {
            tempCS->clearTUs();
          }
          else if( false == swapped )
          {
            tempCS->initStructData( encTestMode.qp, encTestMode.lossless );
            tempCS->copyStructure( *bestCS, partitioner.chType );
            tempCS->getPredBuf().copyFrom( bestCS->getPredBuf() );
            bestCost = bestCS->cost;
            cu = tempCS->getCU( partitioner.chType );
            swapped = true;
          }
          else
          {
            tempCS->clearTUs();
            bestCost = bestCS->cost;
            cu = tempCS->getCU( partitioner.chType );
          }
    
          //we need to restart the distortion for the new tempCS, the bit count and the cost
          tempCS->dist = 0;
          tempCS->fracBits = 0;
          tempCS->cost = MAX_DOUBLE;
          cu->skip = false;
    
          //set SBT info
          cu->setSbtIdx( sbtIdx );
          cu->setSbtPos( sbtPos );
    
          //try residual coding
          m_pcInterSearch->encodeResAndCalcRdInterCU( *tempCS, partitioner, skipResidual );
          numRDOTried++;
    
          xEncodeDontSplit( *tempCS, partitioner );
    
          xCheckDQP( *tempCS, partitioner );
    
          if( imvCS && ( tempCS->cost < imvCS->cost ) )
          {
            if( imvCS->cost != MAX_DOUBLE )
            {
              imvCS->initStructData( encTestMode.qp, encTestMode.lossless );
            }
            imvCS->copyStructure( *tempCS, partitioner.chType );
          }
    
          if( NULL != bestHasNonResi && ( bestCostInternal > tempCS->cost ) )
          {
            bestCostInternal = tempCS->cost;
            if( !( tempCS->getPU( partitioner.chType )->mhIntraFlag ) )
              *bestHasNonResi = !cu->rootCbf;
          }
    
          if( tempCS->cost < currBestCost )
          {
            currBestSbt = cu->sbtInfo;
            currBestTrs = tempCS->tus[cu->sbtInfo ? cu->getSbtPos() : 0]->mtsIdx;
            assert( currBestTrs == 0 || currBestTrs == 1 );
            currBestCost = tempCS->cost;
          }
    
    #if WCG_EXT
          DTRACE_MODE_COST( *tempCS, m_pcRdCost->getLambda( true ) );
    #else
          DTRACE_MODE_COST( *tempCS, m_pcRdCost->getLambda() );
    #endif
          xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );
        }
    
        if( bestCostBegin != bestCS->cost )
        {
          m_sbtCostSave[0] = sbtOffCost;
          m_sbtCostSave[1] = currBestCost;
        }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    
    #if JVET_M0140_SBT
      if( histBestSbt == MAX_UCHAR && doPreAnalyzeResi && numRDOTried > 1 )
      {
        slsSbt->saveBestSbt( cu->cs->area, (uint32_t)( curPuSse >> slShift ), currBestSbt, currBestTrs );
      }
    #endif
    #if JVET_M0140_SBT //harmonize with GBI fast algorithm (move the code here)
      tempCS->cost = currBestCost;
      if( ETM_INTER_ME == encTestMode.type )
      {
        if( equGBiCost != NULL )
        {
          if( tempCS->cost < ( *equGBiCost ) && cu->GBiIdx == GBI_DEFAULT )
          {
            ( *equGBiCost ) = tempCS->cost;
          }
        }
        else
        {
          CHECK( equGBiCost == NULL, "equGBiCost == NULL" );
        }
        if( tempCS->slice->getCheckLDC() && !cu->imv && cu->GBiIdx != GBI_DEFAULT && tempCS->cost < m_bestGbiCost[1] )
        {
          if( tempCS->cost < m_bestGbiCost[0] )
          {
            m_bestGbiCost[1] = m_bestGbiCost[0];
            m_bestGbiCost[0] = tempCS->cost;
            m_bestGbiIdx[1] = m_bestGbiIdx[0];
            m_bestGbiIdx[0] = cu->GBiIdx;
          }
          else
          {
            m_bestGbiCost[1] = tempCS->cost;
            m_bestGbiIdx[1] = cu->GBiIdx;
          }
        }
      }
    #endif
    
    }
    
    
    void EncCu::xEncodeDontSplit( CodingStructure &cs, Partitioner &partitioner )
    {
      m_CABACEstimator->resetBits();
    
    
    Adam Wieckowski's avatar
    Adam Wieckowski committed
    #if JVET_M0421_SPLIT_SIG
      m_CABACEstimator->split_cu_mode( CU_DONT_SPLIT, cs, partitioner );
    #else
    
      {
        if( partitioner.canSplit( CU_QUAD_SPLIT, cs ) )
        {
          m_CABACEstimator->split_cu_flag( false, cs, partitioner );
        }
        if( partitioner.canSplit( CU_MT_SPLIT, cs ) )
        {
          m_CABACEstimator->split_cu_mode_mt( CU_DONT_SPLIT, cs, partitioner );
        }
      }
    
    Adam Wieckowski's avatar
    Adam Wieckowski committed
    #endif
    
    
      cs.fracBits += m_CABACEstimator->getEstFracBits(); // split bits
      cs.cost      = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
    
    }
    
    #if REUSE_CU_RESULTS
    void EncCu::xReuseCachedResult( CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner )
    {
      BestEncInfoCache* bestEncCache = dynamic_cast<BestEncInfoCache*>( m_modeCtrl );
      CHECK( !bestEncCache, "If this mode is chosen, mode controller has to implement the mode caching capabilities" );
      EncTestMode cachedMode;
    
      if( bestEncCache->setCsFrom( *tempCS, cachedMode, partitioner ) )
      {