Skip to content
Snippets Groups Projects
EncCu.cpp 131 KiB
Newer Older
  • Learn to ignore specific revisions
  • Xiaozhong Xu's avatar
    Xiaozhong Xu committed
                if (tempCS->pps->getUseDQP() && (partitioner.currDepth) <= tempCS->pps->getMaxCuDQPDepth())
                {
                  xCheckDQP(*tempCS, partitioner);
                }
    
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
                hasResidual[emtCuFlag] = cu.rootCbf;
                emtCost[emtCuFlag] = tempCS->cost;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
    
                DTRACE_MODE_COST(*tempCS, m_pcRdCost->getLambda());
                xCheckBestMode(tempCS, bestCS, partitioner, encTestMode);
    
                tempCS->initStructData(encTestMode.qp, encTestMode.lossless);
              }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
              if (numResidualPass == 0 && (emtCost[0] <= emtCost[1] ? !hasResidual[0] : !hasResidual[1]))
    
                {
                  // If no residual when allowing for one, then set mark to not try case where residual is forced to 0
                  candHasNoResidual[mergeCand] = 1;
                }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
    
                if (m_pcEncCfg->getUseFastDecisionForMerge() && !bestIsSkip)
                {
                  if (bestCS->getCU(partitioner.chType) == NULL)
                    bestIsSkip = 0;
                  else
                  bestIsSkip = bestCS->getCU(partitioner.chType)->rootCbf == 0;
                }
            }
          }
        }
      }
    
    }
    
    
    Yu Han's avatar
    Yu Han committed
    void EncCu::xCheckRDCostIBCMode(CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner, const EncTestMode& encTestMode)
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
    {
        tempCS->initStructData(encTestMode.qp, encTestMode.lossless);
    
        CodingUnit &cu = tempCS->addCU(CS::getArea(*tempCS, tempCS->area, partitioner.chType), partitioner.chType);
    
        partitioner.setCUData(cu);
        cu.slice = tempCS->slice;
    #if HEVC_TILES_WPP
        cu.tileIdx = tempCS->picture->tileMap->getTileIdxMap(tempCS->area.lumaPos());
    #endif
        cu.skip = false;
        cu.predMode = MODE_INTER;
        cu.transQuantBypass = encTestMode.lossless;
        cu.chromaQpAdj = cu.transQuantBypass ? 0 : m_cuChromaQpOffsetIdxPlus1;
        cu.qp = encTestMode.qp;
    
    Yu Han's avatar
    Yu Han committed
        cu.ibc = true;
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
        cu.imv = 0;
    
        CU::addPUs(cu);
    
        PredictionUnit& pu = *cu.firstPU;
        cu.mmvdSkip = false;
        pu.mmvdMergeFlag = false;
    
    
    Yu Han's avatar
    Yu Han committed
        pu.intraDir[0] = DC_IDX; // set intra pred for ibc block
        pu.intraDir[1] = PLANAR_IDX; // set intra pred for ibc block
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
    
    
    Yu Han's avatar
    Yu Han committed
        pu.interDir = 1; // use list 0 for IBC mode
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
        pu.refIdx[REF_PIC_LIST_0] = pu.cs->slice->getNumRefIdx(REF_PIC_LIST_0) - 1; // last idx in the list
    
    
        if (partitioner.chType == CHANNEL_TYPE_LUMA)
        {
    
    Yu Han's avatar
    Yu Han committed
          bool bValid = m_pcInterSearch->predIBCSearch(cu, partitioner, m_ctuIbcSearchRangeX, m_ctuIbcSearchRangeY, m_ibcHashMap);
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
    
          if (bValid)
          {
            PU::spanMotionInfo(pu);
            const bool chroma = !(CS::isDualITree(*tempCS));
            //  MC
            m_pcInterSearch->motionCompensation(pu, REF_PIC_LIST_0, true, chroma);
    
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if JVET_M0464_UNI_MTS
            {
    #else
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
            double    bestCost = bestCS->cost;
            unsigned char    considerEmtSecondPass = 0;
            bool      skipSecondEmtPass = true;
            double    emtFirstPassCost = MAX_DOUBLE;
    
            // CU-level optimization
    
            for (unsigned char emtCuFlag = 0; emtCuFlag <= considerEmtSecondPass; emtCuFlag++)
            {
              if (m_pcEncCfg->getFastInterEMT() && emtCuFlag && skipSecondEmtPass)
              {
                continue;
              }
    
              tempCS->getCU(tempCS->chType)->emtFlag = emtCuFlag;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
    
              m_pcInterSearch->encodeResAndCalcRdInterCU(*tempCS, partitioner, false, true, chroma);
    
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
              if (m_pcEncCfg->getFastInterEMT())
              {
                emtFirstPassCost = (!emtCuFlag) ? tempCS->cost : emtFirstPassCost;
              }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
              xEncodeDontSplit(*tempCS, partitioner);
    
              if (tempCS->pps->getUseDQP() && (partitioner.currDepth) <= tempCS->pps->getMaxCuDQPDepth())
              {
                xCheckDQP(*tempCS, partitioner);
              }
    
              DTRACE_MODE_COST(*tempCS, m_pcRdCost->getLambda());
              xCheckBestMode(tempCS, bestCS, partitioner, encTestMode);
    
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
              //now we check whether the second pass should be skipped or not
              if (!emtCuFlag && considerEmtSecondPass)
              {
                static const double thresholdToSkipEmtSecondPass = 1.1; // Skip checking EMT transforms
                if (m_pcEncCfg->getFastInterEMT() && (!cu.firstTU->cbf[COMPONENT_Y] || emtFirstPassCost > bestCost * thresholdToSkipEmtSecondPass))
                {
                  skipSecondEmtPass = true;
                }
                else //EMT will be checked
                {
                  if (bestCost == bestCS->cost) //The first EMT pass didn't become the bestCS, so we clear the TUs generated
                  {
                    tempCS->clearTUs();
                  }
                  else
                  {
                    tempCS->initStructData(bestCS->currQP[bestCS->chType], bestCS->isLossless);
    
                    tempCS->copyStructure(*bestCS, partitioner.chType);
                    tempCS->getPredBuf().copyFrom(bestCS->getPredBuf());
                  }
    
                  //we need to restart the distortion for the new tempCS, the bit count and the cost
                  tempCS->dist = 0;
                  tempCS->fracBits = 0;
                  tempCS->cost = MAX_DOUBLE;
                }
              }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
            }
    
          } // bValid
          else
          {
            tempCS->dist = 0;
            tempCS->fracBits = 0;
            tempCS->cost = MAX_DOUBLE;
          }
        }
    
    Yu Han's avatar
    Yu Han committed
     // chroma CU ibc comp
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
        else
        {
          bool success = true;
          // chroma tree, reuse luma bv at minimal block level
          // enabled search only when each chroma sub-block has a BV from its luma sub-block
    
    Yu Han's avatar
    Yu Han committed
          assert(tempCS->getIbcLumaCoverage(pu.Cb()) == IBC_LUMA_COVERAGE_FULL);
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
          // check if each BV for the chroma sub-block is valid
          //static const UInt unitArea = MIN_PU_SIZE * MIN_PU_SIZE;
          const CompArea lumaArea = CompArea(COMPONENT_Y, pu.chromaFormat, pu.Cb().lumaPos(), recalcSize(pu.chromaFormat, CHANNEL_TYPE_CHROMA, CHANNEL_TYPE_LUMA, pu.Cb().size()));
          PredictionUnit subPu;
          subPu.cs = pu.cs;
          subPu.cu = pu.cu;
          const ComponentID compID = COMPONENT_Cb; // use Cb to represent both Cb and CR, as their structures are the same
          int shiftHor = ::getComponentScaleX(compID, pu.chromaFormat);
          int shiftVer = ::getComponentScaleY(compID, pu.chromaFormat);
          //const ChromaFormat  chFmt = pu.chromaFormat;
    
          for (int y = lumaArea.y; y < lumaArea.y + lumaArea.height; y += MIN_PU_SIZE)
          {
            for (int x = lumaArea.x; x < lumaArea.x + lumaArea.width; x += MIN_PU_SIZE)
            {
              const MotionInfo &curMi = pu.cs->picture->cs->getMotionInfo(Position{ x, y });
    
              subPu.UnitArea::operator=(UnitArea(pu.chromaFormat, Area(x, y, MIN_PU_SIZE, MIN_PU_SIZE)));
              Position offsetRef = subPu.blocks[compID].pos().offset((curMi.bv.getHor() >> shiftHor), (curMi.bv.getVer() >> shiftVer));
              Position refEndPos(offsetRef.x + subPu.blocks[compID].size().width - 1, offsetRef.y + subPu.blocks[compID].size().height - 1 );
    
              if (!subPu.cs->isDecomp(refEndPos, toChannelType(compID)) || !subPu.cs->isDecomp(offsetRef, toChannelType(compID))) // ref block is not yet available for this chroma sub-block
              {
                success = false;
                break;
              }
            }
            if (!success)
              break;
          }
          ////////////////////////////////////////////////////////////////////////////
    
          if (success)
          {
    
    Yu Han's avatar
    Yu Han committed
            //pu.mergeType = MRG_TYPE_IBC;
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
            m_pcInterSearch->motionCompensation(pu, REF_PIC_LIST_0, false, true); // luma=0, chroma=1
            m_pcInterSearch->encodeResAndCalcRdInterCU(*tempCS, partitioner, false, false, true);
    
            xEncodeDontSplit(*tempCS, partitioner);
    
            xCheckDQP(*tempCS, partitioner);
    
            DTRACE_MODE_COST(*tempCS, m_pcRdCost->getLambda());
    
            xCheckBestMode(tempCS, bestCS, partitioner, encTestMode);
          }
          else
          {
            tempCS->dist = 0;
            tempCS->fracBits = 0;
            tempCS->cost = MAX_DOUBLE;
          }
        }
      }
    
    Yu Han's avatar
    Yu Han committed
      // check ibc mode in encoder RD
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
      //////////////////////////////////////////////////////////////////////////////////////////////
    
    
    void EncCu::xCheckRDCostInter( CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner, const EncTestMode& encTestMode )
    {
      tempCS->initStructData( encTestMode.qp, encTestMode.lossless );
    
    
      
      m_pcInterSearch->setAffineModeSelected(false);
    
      if( tempCS->slice->getCheckLDC() )
      {
        m_bestGbiCost[0] = m_bestGbiCost[1] = std::numeric_limits<double>::max();
        m_bestGbiIdx[0] = m_bestGbiIdx[1] = -1;
      }
    
      m_pcInterSearch->resetBufferedUniMotions();
      int gbiLoopNum = (tempCS->slice->isInterB() ? GBI_NUM : 1);
      gbiLoopNum = (tempCS->sps->getSpsNext().getUseGBi() ? gbiLoopNum : 1);
    
      if( tempCS->area.lwidth() * tempCS->area.lheight() < GBI_SIZE_CONSTRAINT )
      {
        gbiLoopNum = 1;
      }
    
      double curBestCost = bestCS->cost;
      double equGBiCost = MAX_DOUBLE;
    
      for( int gbiLoopIdx = 0; gbiLoopIdx < gbiLoopNum; gbiLoopIdx++ )
      {
        if( m_pcEncCfg->getUseGBiFast() )
        {
          auto blkCache = dynamic_cast< CacheBlkInfoCtrl* >(m_modeCtrl);
    
          if( blkCache )
          {
            bool isBestInter = blkCache->getInter(bestCS->area);
            uint8_t bestGBiIdx = blkCache->getGbiIdx(bestCS->area);
    
            if( isBestInter && g_GbiSearchOrder[gbiLoopIdx] != GBI_DEFAULT && g_GbiSearchOrder[gbiLoopIdx] != bestGBiIdx )
            {
              continue;
            }
          }
        }
        if( !tempCS->slice->getCheckLDC() )
        {
          if( gbiLoopIdx != 0 && gbiLoopIdx != 3 && gbiLoopIdx != 4 )
          {
            continue;
          }
        }
    
      CodingUnit &cu      = tempCS->addCU( tempCS->area, partitioner.chType );
    
      partitioner.setCUData( cu );
      cu.slice            = tempCS->slice;
    #if HEVC_TILES_WPP
      cu.tileIdx          = tempCS->picture->tileMap->getTileIdxMap( tempCS->area.lumaPos() );
    #endif
      cu.skip             = false;
    
    //cu.affine
      cu.predMode         = MODE_INTER;
      cu.transQuantBypass = encTestMode.lossless;
      cu.chromaQpAdj      = cu.transQuantBypass ? 0 : m_cuChromaQpOffsetIdxPlus1;
      cu.qp               = encTestMode.qp;
      CU::addPUs( cu );
    
    
      cu.GBiIdx = g_GbiSearchOrder[gbiLoopIdx];
      uint8_t gbiIdx = cu.GBiIdx;
      bool  testGbi = (gbiIdx != GBI_DEFAULT);
    
      m_pcInterSearch->predInterSearch( cu, partitioner );
    
      const unsigned wIdx = gp_sizeIdxInfo->idxFrom( tempCS->area.lwidth () );
    
      gbiIdx = CU::getValidGbiIdx(cu);
      if( testGbi && gbiIdx == GBI_DEFAULT ) // Enabled GBi but the search results is uni.
      {
        tempCS->initStructData(encTestMode.qp, encTestMode.lossless);
        continue;
      }
      CHECK(!(testGbi || (!testGbi && gbiIdx == GBI_DEFAULT)), " !( bTestGbi || (!bTestGbi && gbiIdx == GBI_DEFAULT ) )");
    
      bool isEqualUni = false;
      if( m_pcEncCfg->getUseGBiFast() )
      {
        if( cu.firstPU->interDir != 3 && testGbi == 0 )
        {
          isEqualUni = true;
        }
      }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if JVET_M0464_UNI_MTS
      xEncodeInterResidual( tempCS, bestCS, partitioner, encTestMode, 0
                            , m_pImvTempCS ? m_pImvTempCS[wIdx] : NULL
                            , 0
                            , &equGBiCost
    #else
    
      xEncodeInterResidual( tempCS, bestCS, partitioner, encTestMode, 0
    
    Karsten Suehring's avatar
    Karsten Suehring committed
        , m_pImvTempCS ? m_pImvTempCS[wIdx] : NULL
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
      if( g_GbiSearchOrder[gbiLoopIdx] == GBI_DEFAULT )
        m_pcInterSearch->setAffineModeSelected((bestCS->cus.front()->affine && !(bestCS->cus.front()->firstPU->mergeFlag)));
    
      tempCS->initStructData(encTestMode.qp, encTestMode.lossless);
    
      double skipTH = MAX_DOUBLE;
      skipTH = (m_pcEncCfg->getUseGBiFast() ? 1.05 : MAX_DOUBLE);
      if( equGBiCost > curBestCost * skipTH )
      {
        break;
      }
    
      if( m_pcEncCfg->getUseGBiFast() )
      {
        if( isEqualUni == true && m_pcEncCfg->getIntraPeriod() == -1 )
        {
          break;
        }
      }
      if( g_GbiSearchOrder[gbiLoopIdx] == GBI_DEFAULT && xIsGBiSkip(cu) && m_pcEncCfg->getUseGBiFast() )
      {
        break;
      }
     }  // for( UChar gbiLoopIdx = 0; gbiLoopIdx < gbiLoopNum; gbiLoopIdx++ )
    
    }
    
    
    
    
    
    bool EncCu::xCheckRDCostInterIMV( CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner, const EncTestMode& encTestMode )
    {
      int iIMV = int( ( encTestMode.opts & ETO_IMV ) >> ETO_IMV_SHIFT );
    
      m_pcInterSearch->setAffineModeSelected(false);
    
      // Only int-Pel, 4-Pel and fast 4-Pel allowed
      CHECK( iIMV != 1 && iIMV != 2 && iIMV != 3, "Unsupported IMV Mode" );
      // Fast 4-Pel Mode
    
      EncTestMode encTestModeBase = encTestMode;                                        // copy for clearing non-IMV options
      encTestModeBase.opts        = EncTestModeOpts( encTestModeBase.opts & ETO_IMV );  // clear non-IMV options (is that intended?)
    
      tempCS->initStructData( encTestMode.qp, encTestMode.lossless );
    
      CodingStructure* pcCUInfo2Reuse = nullptr;
    
    
      m_pcInterSearch->resetBufferedUniMotions();
      int gbiLoopNum = (tempCS->slice->isInterB() ? GBI_NUM : 1);
      gbiLoopNum = (pcCUInfo2Reuse != NULL ? 1 : gbiLoopNum);
      gbiLoopNum = (tempCS->slice->getSPS()->getSpsNext().getUseGBi() ? gbiLoopNum : 1);
    
      if( tempCS->area.lwidth() * tempCS->area.lheight() < GBI_SIZE_CONSTRAINT )
      {
        gbiLoopNum = 1;
      }
    
      double curBestCost = bestCS->cost;
      double equGBiCost = MAX_DOUBLE;
    
      for( int gbiLoopIdx = 0; gbiLoopIdx < gbiLoopNum; gbiLoopIdx++ )
      {
        if( m_pcEncCfg->getUseGBiFast() )
        {
          auto blkCache = dynamic_cast< CacheBlkInfoCtrl* >(m_modeCtrl);
    
          if( blkCache )
          {
            bool isBestInter = blkCache->getInter(bestCS->area);
            uint8_t bestGBiIdx = blkCache->getGbiIdx(bestCS->area);
    
            if( isBestInter && g_GbiSearchOrder[gbiLoopIdx] != GBI_DEFAULT && g_GbiSearchOrder[gbiLoopIdx] != bestGBiIdx )
            {
              continue;
            }
          }
        }
    
        if( !tempCS->slice->getCheckLDC() )
        {
          if( gbiLoopIdx != 0 && gbiLoopIdx != 3 && gbiLoopIdx != 4 )
          {
            continue;
          }
        }
    
        if( m_pcEncCfg->getUseGBiFast() && tempCS->slice->getCheckLDC() && g_GbiSearchOrder[gbiLoopIdx] != GBI_DEFAULT
          && (m_bestGbiIdx[0] >= 0 && g_GbiSearchOrder[gbiLoopIdx] != m_bestGbiIdx[0])
          && (m_bestGbiIdx[1] >= 0 && g_GbiSearchOrder[gbiLoopIdx] != m_bestGbiIdx[1]))
        {
          continue;
        }
    
      CodingUnit &cu = ( pcCUInfo2Reuse != nullptr ) ? *tempCS->getCU( partitioner.chType ) : tempCS->addCU( tempCS->area, partitioner.chType );
    
      if( pcCUInfo2Reuse == nullptr )
      {
        partitioner.setCUData( cu );
        cu.slice            = tempCS->slice;
    #if HEVC_TILES_WPP
        cu.tileIdx          = tempCS->picture->tileMap->getTileIdxMap( tempCS->area.lumaPos() );
    #endif
        cu.skip             = false;
    
      //cu.affine
        cu.predMode         = MODE_INTER;
        cu.transQuantBypass = encTestMode.lossless;
        cu.chromaQpAdj      = cu.transQuantBypass ? 0 : m_cuChromaQpOffsetIdxPlus1;
        cu.qp               = encTestMode.qp;
    
        CU::addPUs( cu );
      }
      else
      {
        CHECK( cu.skip,                                "Mismatch" );
        CHECK( cu.qtDepth  != partitioner.currQtDepth, "Mismatch" );
        CHECK( cu.btDepth  != partitioner.currBtDepth, "Mismatch" );
        CHECK( cu.mtDepth  != partitioner.currMtDepth, "Mismatch" );
        CHECK( cu.depth    != partitioner.currDepth,   "Mismatch" );
      }
    
      cu.imv      = iIMV > 1 ? 2 : 1;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
      bool testGbi;
      uint8_t gbiIdx;
    
      if( pcCUInfo2Reuse != nullptr )
      {
        // reuse the motion info from pcCUInfo2Reuse
        CU::resetMVDandMV2Int( cu, m_pcInterSearch );
    
    
        CHECK(cu.GBiIdx < 0 || cu.GBiIdx >= GBI_NUM, "cu.GBiIdx < 0 || cu.GBiIdx >= GBI_NUM");
        gbiIdx = CU::getValidGbiIdx(cu);
        testGbi = (gbiIdx != GBI_DEFAULT);
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
          if (m_modeCtrl->useModeResult(encTestModeBase, tempCS, partitioner))
          {
            std::swap(tempCS, bestCS);
            // store temp best CI for next CU coding
            m_CurrCtx->best = m_CABACEstimator->getCtx();
          }
    
        cu.GBiIdx = g_GbiSearchOrder[gbiLoopIdx];
        gbiIdx = cu.GBiIdx;
        testGbi = (gbiIdx != GBI_DEFAULT);
    
        m_pcInterSearch->predInterSearch( cu, partitioner );
    
        gbiIdx = CU::getValidGbiIdx(cu);
    
      if( testGbi && gbiIdx == GBI_DEFAULT ) // Enabled GBi but the search results is uni.
      {
        tempCS->initStructData(encTestMode.qp, encTestMode.lossless);
        continue;
      }
      CHECK(!(testGbi || (!testGbi && gbiIdx == GBI_DEFAULT)), " !( bTestGbi || (!bTestGbi && gbiIdx == GBI_DEFAULT ) )");
    
      bool isEqualUni = false;
      if( m_pcEncCfg->getUseGBiFast() )
      {
        if( cu.firstPU->interDir != 3 && testGbi == 0 )
        {
          isEqualUni = true;
        }
      }
    
    Xiaozhong Xu's avatar
    Xiaozhong Xu committed
        if (m_modeCtrl->useModeResult(encTestModeBase, tempCS, partitioner))
        {
          std::swap(tempCS, bestCS);
          // store temp best CI for next CU coding
          m_CurrCtx->best = m_CABACEstimator->getCtx();
        }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if JVET_M0464_UNI_MTS
      xEncodeInterResidual( tempCS, bestCS, partitioner, encTestModeBase, 0
                            , NULL
                            , 0
                            , &equGBiCost
    #else
    
      xEncodeInterResidual( tempCS, bestCS, partitioner, encTestModeBase, 0
        , NULL
        , true
        , 0
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
      tempCS->initStructData(encTestMode.qp, encTestMode.lossless);
    
      double skipTH = MAX_DOUBLE;
      skipTH = (m_pcEncCfg->getUseGBiFast() ? 1.05 : MAX_DOUBLE);
      if( equGBiCost > curBestCost * skipTH )
      {
        break;
      }
    
      if( m_pcEncCfg->getUseGBiFast() )
      {
        if( isEqualUni == true && m_pcEncCfg->getIntraPeriod() == -1 )
        {
          break;
        }
      }
      if( g_GbiSearchOrder[gbiLoopIdx] == GBI_DEFAULT && xIsGBiSkip(cu) && m_pcEncCfg->getUseGBiFast() )
      {
        break;
      }
     } // for( UChar gbiLoopIdx = 0; gbiLoopIdx < gbiLoopNum; gbiLoopIdx++ )
    
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if JVET_M0464_UNI_MTS
    void EncCu::xEncodeInterResidual(   CodingStructure *&tempCS
                                      , CodingStructure *&bestCS
                                      , Partitioner &partitioner
                                      , const EncTestMode& encTestMode
                                      , int residualPass
                                      , CodingStructure* imvCS
                                      , bool* bestHasNonResi
                                      , double* equGBiCost
    #else
    
    void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner, const EncTestMode& encTestMode, int residualPass
      , CodingStructure* imvCS
      , int emtMode
      , bool* bestHasNonResi
    
      , double* equGBiCost
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    {
      if( residualPass == 1 && encTestMode.lossless )
      {
        return;
      }
    
      CodingUnit*            cu        = tempCS->getCU( partitioner.chType );
      double   bestCostInternal        = MAX_DOUBLE;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Karsten Suehring's avatar
    Karsten Suehring committed
      const int      maxSizeEMT        = EMT_INTER_MAX_CU_WITH_QTBT;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
      bool              swapped        = false; // avoid unwanted data copy
      bool             reloadCU        = false;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
      const bool considerEmtSecondPass = emtMode && sps.getSpsNext().getUseInterEMT() && partitioner.currArea().lwidth() <= maxSizeEMT && partitioner.currArea().lheight() <= maxSizeEMT;
    
      int minEMTMode = 0;
      int maxEMTMode = (considerEmtSecondPass?1:0);
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    Xiang Li's avatar
    Xiang Li committed
      // Not allow very big |MVd| to avoid CABAC crash caused by too large MVd. Normally no impact on coding performance.
      const int maxMvd = 1 << 15;
      const PredictionUnit& pu = *cu->firstPU;
      if (!cu->affine)
      {
        if ((pu.refIdx[0] >= 0 && (pu.mvd[0].getAbsHor() >= maxMvd || pu.mvd[0].getAbsVer() >= maxMvd))
          || (pu.refIdx[1] >= 0 && (pu.mvd[1].getAbsHor() >= maxMvd || pu.mvd[1].getAbsVer() >= maxMvd)))
        {
          return;
        }
      }
    
      else
      {
        for (int refList = 0; refList < NUM_REF_PIC_LIST_01; refList++)
        {
          if (pu.refIdx[refList] >= 0)
          {
            for (int ctrlP = 1 + (cu->affineType == AFFINEMODEL_6PARAM); ctrlP >= 0; ctrlP--)
            {
              if (pu.mvdAffi[refList][ctrlP].getAbsHor() >= maxMvd || pu.mvdAffi[refList][ctrlP].getAbsVer() >= maxMvd)
              {
                return;
              }
            }
          }
        }
      }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
      if( emtMode == 2 )
      {
        minEMTMode = maxEMTMode = (cu->emtFlag?1:0);
      }
    
      for( int curEmtMode = minEMTMode; curEmtMode <= maxEMTMode; curEmtMode++ )
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
        if( reloadCU )
        {
          if( bestCost == bestCS->cost ) //The first EMT pass didn't become the bestCS, so we clear the TUs generated
          {
            tempCS->clearTUs();
          }
          else if( false == swapped )
          {
            tempCS->initStructData( encTestMode.qp, encTestMode.lossless );
            tempCS->copyStructure( *bestCS, partitioner.chType );
            tempCS->getPredBuf().copyFrom( bestCS->getPredBuf() );
            bestCost = bestCS->cost;
            cu       = tempCS->getCU( partitioner.chType );
            swapped = true;
          }
          else
          {
            tempCS->clearTUs();
            bestCost = bestCS->cost;
            cu       = tempCS->getCU( partitioner.chType );
          }
    
          //we need to restart the distortion for the new tempCS, the bit count and the cost
          tempCS->dist     = 0;
          tempCS->fracBits = 0;
          tempCS->cost     = MAX_DOUBLE;
        }
    
        reloadCU    = true; // enable cu reloading
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    
        const bool skipResidual = residualPass == 1;
        m_pcInterSearch->encodeResAndCalcRdInterCU( *tempCS, partitioner, skipResidual );
    
        xEncodeDontSplit( *tempCS, partitioner );
    
        xCheckDQP( *tempCS, partitioner );
    
    
        if( ETM_INTER_ME == encTestMode.type )
        {
          if( equGBiCost != NULL )
          {
            if( tempCS->cost < (*equGBiCost) && cu->GBiIdx == GBI_DEFAULT )
            {
              (*equGBiCost) = tempCS->cost;
            }
          }
          else
          {
            CHECK(equGBiCost == NULL, "equGBiCost == NULL");
          }
          if( tempCS->slice->getCheckLDC() && !cu->imv && cu->GBiIdx != GBI_DEFAULT && tempCS->cost < m_bestGbiCost[1] )
          {
            if( tempCS->cost < m_bestGbiCost[0] )
            {
              m_bestGbiCost[1] = m_bestGbiCost[0];
              m_bestGbiCost[0] = tempCS->cost;
              m_bestGbiIdx[1] = m_bestGbiIdx[0];
              m_bestGbiIdx[0] = cu->GBiIdx;
            }
            else
            {
              m_bestGbiCost[1] = tempCS->cost;
              m_bestGbiIdx[1] = cu->GBiIdx;
            }
          }
        }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
        double emtFirstPassCost = tempCS->cost;
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
        if( imvCS && (tempCS->cost < imvCS->cost) )
        {
          if( imvCS->cost != MAX_DOUBLE )
          {
            imvCS->initStructData( encTestMode.qp, encTestMode.lossless );
          }
          imvCS->copyStructure( *tempCS, partitioner.chType );
        }
        if( NULL != bestHasNonResi && (bestCostInternal > tempCS->cost) )
        {
          bestCostInternal = tempCS->cost;
    
          if (!(tempCS->getPU(partitioner.chType)->mhIntraFlag))
    
          if (tempCS->getPU(partitioner.chType)->mhIntraFlag)
    
    #if WCG_EXT
        DTRACE_MODE_COST( *tempCS, m_pcRdCost->getLambda( true ) );
    #else
        DTRACE_MODE_COST( *tempCS, m_pcRdCost->getLambda() );
    #endif
        xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );
    
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #if !JVET_M0464_UNI_MTS
    
        //now we check whether the second pass should be skipped or not
        if( !curEmtMode && maxEMTMode )
        {
          const double thresholdToSkipEmtSecondPass = 1.1; // Skip checking EMT transforms
          const bool bCond1 = !cu->firstTU->cbf[COMPONENT_Y];
    
          const bool bCond3 = emtFirstPassCost > ( bestCost * thresholdToSkipEmtSecondPass );
    
          if( m_pcEncCfg->getFastInterEMT() && (bCond1 || bCond3 ) ) 
          {
            maxEMTMode = 0; // do not test EMT
          }
        }
    
    Tung Nguyen's avatar
    Tung Nguyen committed
    #endif
    
    }
    
    
    void EncCu::xEncodeDontSplit( CodingStructure &cs, Partitioner &partitioner )
    {
      m_CABACEstimator->resetBits();
    
    
    Adam Wieckowski's avatar
    Adam Wieckowski committed
    #if JVET_M0421_SPLIT_SIG
      m_CABACEstimator->split_cu_mode( CU_DONT_SPLIT, cs, partitioner );
    #else
    
      {
        if( partitioner.canSplit( CU_QUAD_SPLIT, cs ) )
        {
          m_CABACEstimator->split_cu_flag( false, cs, partitioner );
        }
        if( partitioner.canSplit( CU_MT_SPLIT, cs ) )
        {
          m_CABACEstimator->split_cu_mode_mt( CU_DONT_SPLIT, cs, partitioner );
        }
      }
    
    Adam Wieckowski's avatar
    Adam Wieckowski committed
    #endif
    
    
      cs.fracBits += m_CABACEstimator->getEstFracBits(); // split bits
      cs.cost      = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
    
    }
    
    #if REUSE_CU_RESULTS
    void EncCu::xReuseCachedResult( CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner )
    {
      const SPS &sps = *tempCS->sps;
    
      BestEncInfoCache* bestEncCache = dynamic_cast<BestEncInfoCache*>( m_modeCtrl );
      CHECK( !bestEncCache, "If this mode is chosen, mode controller has to implement the mode caching capabilities" );
      EncTestMode cachedMode;
    
      if( bestEncCache->setCsFrom( *tempCS, cachedMode, partitioner ) )
      {
        CodingUnit& cu = *tempCS->cus.front();
    
    #if JVET_M0170_MRG_SHARELIST
        cu.shareParentPos = tempCS->sharedBndPos;
        cu.shareParentSize = tempCS->sharedBndSize;
    #endif
    
        partitioner.setCUData( cu );
    
        if( CU::isIntra( cu ) )
        {
          xReconIntraQT( cu );
        }
        else
        {
          xDeriveCUMV( cu );
          xReconInter( cu );
        }
    
        Distortion finalDistortion = 0;
        const int  numValidComponents = getNumberValidComponents( tempCS->area.chromaFormat );
    
        for( int comp = 0; comp < numValidComponents; comp++ )
        {
          const ComponentID compID = ComponentID( comp );
    
          if( CS::isDualITree( *tempCS ) && toChannelType( compID ) != partitioner.chType )
          {
            continue;
          }
    
          CPelBuf reco = tempCS->getRecoBuf( compID );
          CPelBuf org  = tempCS->getOrgBuf ( compID );
    
    #if WCG_EXT
          if( m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled() )
          {
            const CPelBuf orgLuma = tempCS->getOrgBuf(tempCS->area.blocks[COMPONENT_Y]);
            finalDistortion += m_pcRdCost->getDistPart( org, reco, sps.getBitDepth( toChannelType( compID ) ), compID, DF_SSE_WTD, &orgLuma );
          }
          else
    #endif
          finalDistortion += m_pcRdCost->getDistPart( org, reco, sps.getBitDepth( toChannelType( compID ) ), compID, DF_SSE );
        }
    
        m_CABACEstimator->getCtx() = m_CurrCtx->start;
        m_CABACEstimator->resetBits();
    
        CUCtx cuCtx;
        cuCtx.isDQPCoded = true;
        cuCtx.isChromaQpAdjCoded = true;
        m_CABACEstimator->coding_unit( cu, partitioner, cuCtx );
    
        tempCS->dist     = finalDistortion;
        tempCS->fracBits = m_CABACEstimator->getEstFracBits();
        tempCS->cost     = m_pcRdCost->calcRdCost( tempCS->fracBits, tempCS->dist );
    
        xEncodeDontSplit( *tempCS,         partitioner );
        xCheckDQP       ( *tempCS,         partitioner );
        xCheckBestMode  (  tempCS, bestCS, partitioner, cachedMode );
      }
      else
      {
        THROW( "Should never happen!" );
      }
    }
    
    #endif