Skip to content
Snippets Groups Projects
IntraSearch.cpp 483 KiB
Newer Older
  • Learn to ignore specific revisions
  • /* The copyright in this software is being made available under the BSD
     * License, included below. This software may be subject to other third party
     * and contributor rights, including patent rights, and no such rights are
     * granted under this license.
     *
    
     * Copyright (c) 2010-2023, ITU/ISO/IEC
    
     * All rights reserved.
     *
     * Redistribution and use in source and binary forms, with or without
     * modification, are permitted provided that the following conditions are met:
     *
     *  * Redistributions of source code must retain the above copyright notice,
     *    this list of conditions and the following disclaimer.
     *  * Redistributions in binary form must reproduce the above copyright notice,
     *    this list of conditions and the following disclaimer in the documentation
     *    and/or other materials provided with the distribution.
     *  * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
     *    be used to endorse or promote products derived from this software without
     *    specific prior written permission.
     *
     * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
     * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
     * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
     * THE POSSIBILITY OF SUCH DAMAGE.
     */
    
    /** \file     EncSearch.cpp
     *  \brief    encoder intra search class
     */
    
    #include "IntraSearch.h"
    
    #include "EncModeCtrl.h"
    
    #include "CommonLib/CommonDef.h"
    #include "CommonLib/Rom.h"
    #include "CommonLib/Picture.h"
    #include "CommonLib/UnitTools.h"
    
    #if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #include "CommonLib/BilateralFilter.h"
    #endif
    
    
    #include "CommonLib/dtrace_next.h"
    #include "CommonLib/dtrace_buffer.h"
    
    #include <math.h>
    #include <limits>
     //! \ingroup EncoderLib
     //! \{
    
    IntraSearch::IntraSearch()
      : m_pSplitCS      (nullptr)
      , m_pFullCS       (nullptr)
      , m_pBestCS       (nullptr)
      , m_pcEncCfg      (nullptr)
    
    #if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      , m_bilateralFilter(nullptr)
    #endif
    
      , m_pcTrQuant     (nullptr)
      , m_pcRdCost      (nullptr)
    
    Taoran Lu's avatar
    Taoran Lu committed
      , m_pcReshape     (nullptr)
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      , m_ctxCache      (nullptr)
    
      , m_isInitialized (false)
    {
      for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
      {
        m_pSharedPredTransformSkip[ch] = nullptr;
      }
    
    fan wang's avatar
    fan wang committed
    #if JVET_AB0155_SGPM
    
    #if JVET_AG0152_SGPM_ITMP_IBC
      for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
    #else
    
    fan wang's avatar
    fan wang committed
      for (int i = 0; i < NUM_LUMA_MODE; i++)
    
    fan wang's avatar
    fan wang committed
      {
        m_intraPredBuf[i] = nullptr;
      }
      for (int i = 0; i < SGPM_NUM; i++)
      {
        m_sgpmPredBuf[i] = nullptr;
      }
    #endif
    
    #if JVET_AG0058_EIP
      for (int i = 0; i < NUM_DERIVED_EIP; i++)
      {
        m_eipPredBuf[i] = nullptr;
      }
      for(int i = 0; i < MAX_MERGE_EIP;i++)
      {
        m_eipMergePredBuf[i] = nullptr;
      }
    
    #endif
    #if JVET_AH0076_OBIC
      m_dimdPredBuf = nullptr;
      m_obicPredBuf = nullptr;
    
      m_truncBinBits = nullptr;
      m_escapeNumBins = nullptr;
      m_minErrorIndexMap = nullptr;
      for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
      {
        m_indexError[i] = nullptr;
      }
      for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
      {
        m_statePtRDOQ[i] = nullptr;
      }
    
    
      ::memset( m_indexMapRDOQ, 0, sizeof( m_indexMapRDOQ ) );
      ::memset( m_runMapRDOQ, false, sizeof( m_runMapRDOQ ) );
      ::memset( m_prevRunTypeRDOQ, false, sizeof( m_prevRunTypeRDOQ ) );
      ::memset( m_prevRunPosRDOQ, 0, sizeof( m_prevRunPosRDOQ ) );
      ::memset( m_stateCostRDOQ, 0, sizeof( m_stateCostRDOQ ) );
    
    }
    
    
    void IntraSearch::destroy()
    {
      CHECK( !m_isInitialized, "Not initialized" );
    
      if( m_pcEncCfg )
      {
    
    Karsten Suehring's avatar
    Karsten Suehring committed
        const uint32_t uiNumLayersToAllocateSplit = 1;
        const uint32_t uiNumLayersToAllocateFull  = 1;
    
        const int uiNumSaveLayersToAllocate = 2;
    
        for( uint32_t layer = 0; layer < uiNumSaveLayersToAllocate; layer++ )
        {
          m_pSaveCS[layer]->destroy();
          delete m_pSaveCS[layer];
        }
    
        uint32_t numWidths  = gp_sizeIdxInfo->numWidths();
        uint32_t numHeights = gp_sizeIdxInfo->numHeights();
    
        for( uint32_t width = 0; width < numWidths; width++ )
        {
          for( uint32_t height = 0; height < numHeights; height++ )
          {
    
            if( gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( width ) ) && gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( height ) )
              && gp_sizeIdxInfo->sizeFrom(width) <= m_pcEncCfg->getMaxCUWidth() && gp_sizeIdxInfo->sizeFrom(height) <= m_pcEncCfg->getMaxCUHeight())
    
            {
              for( uint32_t layer = 0; layer < uiNumLayersToAllocateSplit; layer++ )
              {
                m_pSplitCS[width][height][layer]->destroy();
    
                delete m_pSplitCS[width][height][layer];
              }
    
              for( uint32_t layer = 0; layer < uiNumLayersToAllocateFull; layer++ )
              {
                m_pFullCS[width][height][layer]->destroy();
    
                delete m_pFullCS[width][height][layer];
              }
    
              delete[] m_pSplitCS[width][height];
              delete[] m_pFullCS [width][height];
    
              m_pBestCS[width][height]->destroy();
              m_pTempCS[width][height]->destroy();
    
              delete m_pTempCS[width][height];
              delete m_pBestCS[width][height];
            }
          }
    
          delete[] m_pSplitCS[width];
          delete[] m_pFullCS [width];
    
          delete[] m_pTempCS[width];
          delete[] m_pBestCS[width];
        }
    
        delete[] m_pSplitCS;
        delete[] m_pFullCS;
    
        delete[] m_pBestCS;
        delete[] m_pTempCS;
    
        delete[] m_pSaveCS;
      }
    
      m_pSplitCS = m_pFullCS = nullptr;
    
      m_pBestCS = m_pTempCS = nullptr;
    
      m_pSaveCS = nullptr;
    
      for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
      {
        delete[] m_pSharedPredTransformSkip[ch];
        m_pSharedPredTransformSkip[ch] = nullptr;
      }
    
    
    #if JVET_AC0119_LM_CHROMA_FUSION
      for (uint32_t i = 0; i < 2; i++)
      {
        m_predStorage[i].destroy();
      }
      for (uint32_t i = 0; i < 6; i++)
      {
        m_fusionStorage[i].destroy();
      }
    #endif
    
    
    #if JVET_AD0120_LBCCP
      for (uint32_t i = 0; i < LBCCP_FILTER_MMLMNUM; i++)
      {
        m_lmPredFiltStorage[i].destroy();
      }
    #endif
    
    
    #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
      for (uint32_t i = 0; i < 2; i++)
      {
        m_predCCPFusionStorage[i].destroy();
      }
    #endif
    
    
    #if JVET_AB0143_CCCM_TS
    
    #if JVET_AD0202_CCCM_MDF
      for (uint32_t cccmIdx = 0; cccmIdx < TOTAL_NUM_CCCM_MODES; cccmIdx++)
    #else
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      for (uint32_t cccmIdx = 0; cccmIdx < CCCM_NUM_MODES; cccmIdx++)
    
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING
        m_cccmStorage[0][cccmIdx].destroy();
        m_cccmStorage[1][cccmIdx].destroy();
    #else
    
        m_cccmStorage[cccmIdx].destroy();
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
      m_ddCcpStorage.destroy();
      for (int i = 0; i < 2; i++)
      {
        m_ddCcpFusionStorage[i].destroy();
      }
    #endif
    
    Taoran Lu's avatar
    Taoran Lu committed
      m_tmpStorageLCU.destroy();
    
      m_colorTransResiBuf.destroy();
    
    fan wang's avatar
    fan wang committed
      
    #if JVET_AB0155_SGPM
    
    #if JVET_AG0152_SGPM_ITMP_IBC
      for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
    #else
    
    fan wang's avatar
    fan wang committed
      for (int i = 0; i < NUM_LUMA_MODE; i++)
    
    fan wang's avatar
    fan wang committed
      {
        delete[] m_intraPredBuf[i];
        m_intraPredBuf[i] = nullptr;
      }
    
      for (int i = 0; i < SGPM_NUM; i++)
      {
        delete[] m_sgpmPredBuf[i];
        m_sgpmPredBuf[i] = nullptr;
      }
    #endif
    
    #if JVET_AG0058_EIP
      for (int i = 0; i < NUM_DERIVED_EIP; i++)
      {
        delete[] m_eipPredBuf[i];
        m_eipPredBuf[i] = nullptr;
      }
      for(int i = 0; i < MAX_MERGE_EIP;i++)
      {
        delete[] m_eipMergePredBuf[i];
        m_eipMergePredBuf[i] = nullptr;
      }
    
    #endif
    #if JVET_AH0076_OBIC
      delete[] m_dimdPredBuf;
      m_dimdPredBuf = nullptr;
      delete[] m_obicPredBuf;
      m_obicPredBuf = nullptr;
    
      if (m_truncBinBits != nullptr)
      {
        for (unsigned i = 0; i < m_symbolSize; i++)
        {
          delete[] m_truncBinBits[i];
          m_truncBinBits[i] = nullptr;
        }
        delete[] m_truncBinBits;
        m_truncBinBits = nullptr;
      }
      if (m_escapeNumBins != nullptr)
      {
        delete[] m_escapeNumBins;
        m_escapeNumBins = nullptr;
      }
      if (m_indexError[0] != nullptr)
      {
        for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
        {
          delete[] m_indexError[i];
          m_indexError[i] = nullptr;
        }
      }
      if (m_minErrorIndexMap != nullptr)
      {
        delete[] m_minErrorIndexMap;
        m_minErrorIndexMap = nullptr;
      }
      if (m_statePtRDOQ[0] != nullptr)
      {
        for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
        {
          delete[] m_statePtRDOQ[i];
          m_statePtRDOQ[i] = nullptr;
        }
      }
    
    }
    
    IntraSearch::~IntraSearch()
    {
      if( m_isInitialized )
      {
        destroy();
      }
    }
    
    void IntraSearch::init( EncCfg*        pcEncCfg,
    
    #if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER
    
    Vadim Seregin's avatar
    Vadim Seregin committed
                           BilateralFilter* bilateralFilter,
    #endif
    
                            TrQuant*       pcTrQuant,
                            RdCost*        pcRdCost,
                            CABACWriter*   CABACEstimator,
                            CtxCache*      ctxCache,
                            const uint32_t     maxCUWidth,
                            const uint32_t     maxCUHeight,
                            const uint32_t     maxTotalCUDepth
    
    Taoran Lu's avatar
    Taoran Lu committed
                           , EncReshape*   pcReshape
    
                           , const unsigned bitDepthY
    
    )
    {
      CHECK(m_isInitialized, "Already initialized");
      m_pcEncCfg                     = pcEncCfg;
    
    #if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      m_bilateralFilter              = bilateralFilter;
    #endif
    
      m_pcTrQuant                    = pcTrQuant;
      m_pcRdCost                     = pcRdCost;
      m_CABACEstimator               = CABACEstimator;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      m_ctxCache                     = ctxCache;
    
    Taoran Lu's avatar
    Taoran Lu committed
      m_pcReshape                    = pcReshape;
    
    
      const ChromaFormat cform = pcEncCfg->getChromaFormatIdc();
    
      IntraPrediction::init( cform, pcEncCfg->getBitDepth( CHANNEL_TYPE_LUMA ) );
    
      m_tmpStorageLCU.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
      m_colorTransResiBuf.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
    
    #if JVET_AC0119_LM_CHROMA_FUSION
      for (uint32_t i = 0; i < 2; i++)
      {
    
        m_predStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
    
      }
      for (uint32_t i = 0; i < 6; i++)
      {
    
        m_fusionStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
    
    #if JVET_AD0120_LBCCP
      for (uint32_t i = 0; i < LBCCP_FILTER_MMLMNUM; i++)
      {
        m_lmPredFiltStorage[i].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
      }
    #endif
    
    
    #if JVET_AG0059_CCP_MERGE_ENHANCEMENT
      for (uint32_t i = 0; i < 2; i++)
      {
        m_predCCPFusionStorage[i].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
      }
    #endif
    
    
    #if JVET_AB0143_CCCM_TS
    
    #if JVET_AD0202_CCCM_MDF
      for (uint32_t cccmIdx = 0; cccmIdx < TOTAL_NUM_CCCM_MODES; cccmIdx++)
    #else
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      for (uint32_t cccmIdx = 0; cccmIdx < CCCM_NUM_MODES; cccmIdx++)
    
    #if JVET_AC0147_CCCM_NO_SUBSAMPLING
    
        m_cccmStorage[0][cccmIdx].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
        m_cccmStorage[1][cccmIdx].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
    
        m_cccmStorage[cccmIdx].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
    
    #if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
      m_ddCcpStorage.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
      for (int i = 0; i < 2; i++)
      {
        m_ddCcpFusionStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
      }
    #endif
    
    #if JVET_AB0155_SGPM || JVET_AH0076_OBIC
    
    #if JVET_AG0152_SGPM_ITMP_IBC
      for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
    #else
    
    fan wang's avatar
    fan wang committed
      for (int i = 0; i < NUM_LUMA_MODE; i++)
    
    fan wang's avatar
    fan wang committed
      {
    
    #if JVET_AH0076_OBIC
        m_intraPredBuf[i] = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
    #else
    
    fan wang's avatar
    fan wang committed
        m_intraPredBuf[i] = new Pel[GEO_MAX_CU_SIZE_EX * GEO_MAX_CU_SIZE_EX];
    
    fan wang's avatar
    fan wang committed
      }
      for (int i = 0; i < SGPM_NUM; i++)
      {
        m_sgpmPredBuf[i] = new Pel[GEO_MAX_CU_SIZE_EX * GEO_MAX_CU_SIZE_EX];
      }
    #endif
    
    #if JVET_AG0058_EIP
      for (int i = 0; i < NUM_DERIVED_EIP; i++)
      {
        m_eipPredBuf[i] = new Pel[MAX_EIP_SIZE * MAX_EIP_SIZE];
      }
      for(int i = 0; i < MAX_MERGE_EIP;i++)
      {
        m_eipMergePredBuf[i] = new Pel[MAX_EIP_SIZE * MAX_EIP_SIZE];
      }
    
    #endif
    #if JVET_AH0076_OBIC
      m_dimdPredBuf = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
      m_obicPredBuf = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
    
      for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
      {
    
        m_pSharedPredTransformSkip[ch] = new Pel[maxCUWidth * maxCUHeight];
    
      }
    
      uint32_t numWidths  = gp_sizeIdxInfo->numWidths();
      uint32_t numHeights = gp_sizeIdxInfo->numHeights();
    
    
    Karsten Suehring's avatar
    Karsten Suehring committed
      const uint32_t uiNumLayersToAllocateSplit = 1;
      const uint32_t uiNumLayersToAllocateFull  = 1;
    
    
      m_pBestCS = new CodingStructure**[numWidths];
      m_pTempCS = new CodingStructure**[numWidths];
    
      m_pFullCS  = new CodingStructure***[numWidths];
      m_pSplitCS = new CodingStructure***[numWidths];
    
      for( uint32_t width = 0; width < numWidths; width++ )
      {
        m_pBestCS[width] = new CodingStructure*[numHeights];
        m_pTempCS[width] = new CodingStructure*[numHeights];
    
        m_pFullCS [width] = new CodingStructure**[numHeights];
        m_pSplitCS[width] = new CodingStructure**[numHeights];
    
        for( uint32_t height = 0; height < numHeights; height++ )
        {
    
          if(  gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( width ) ) && gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( height ) )
            && gp_sizeIdxInfo->sizeFrom(width) <= maxCUWidth && gp_sizeIdxInfo->sizeFrom(height) <= maxCUHeight)
    
          {
            m_pBestCS[width][height] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
            m_pTempCS[width][height] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
    
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #if JVET_Z0118_GDR
            m_pBestCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
            m_pTempCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
    #else
    
            m_pBestCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
            m_pTempCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #endif
    
            m_pFullCS [width][height] = new CodingStructure*[uiNumLayersToAllocateFull];
            m_pSplitCS[width][height] = new CodingStructure*[uiNumLayersToAllocateSplit];
    
            for( uint32_t layer = 0; layer < uiNumLayersToAllocateFull; layer++ )
            {
              m_pFullCS [width][height][layer] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
    
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #if JVET_Z0118_GDR
              m_pFullCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
    #else
    
              m_pFullCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #endif
    
            }
    
            for( uint32_t layer = 0; layer < uiNumLayersToAllocateSplit; layer++ )
            {
              m_pSplitCS[width][height][layer] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #if JVET_Z0118_GDR
              m_pSplitCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
    #else
    
              m_pSplitCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #endif
    
            }
          }
          else
          {
            m_pBestCS[width][height] = nullptr;
            m_pTempCS[width][height] = nullptr;
    
            m_pFullCS [width][height] = nullptr;
            m_pSplitCS[width][height] = nullptr;
          }
        }
      }
    
      const int uiNumSaveLayersToAllocate = 2;
    
      m_pSaveCS = new CodingStructure*[uiNumSaveLayersToAllocate];
    
      for( uint32_t depth = 0; depth < uiNumSaveLayersToAllocate; depth++ )
      {
        m_pSaveCS[depth] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #if JVET_Z0118_GDR
        m_pSaveCS[depth]->create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
    #else
    
        m_pSaveCS[depth]->create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)), false, (bool)pcEncCfg->getPLTMode());
    
    Seungwook Hong's avatar
    Seungwook Hong committed
    #endif
    
      if (pcEncCfg->getPLTMode())
    
        m_symbolSize = (1 << bitDepthY); // pixel values are within [0, SymbolSize-1] with size SymbolSize
        if (m_truncBinBits == nullptr)
    
          m_truncBinBits = new uint16_t*[m_symbolSize];
          for (unsigned i = 0; i < m_symbolSize; i++)
          {
            m_truncBinBits[i] = new uint16_t[m_symbolSize + 1];
          }
    
        if (m_escapeNumBins == nullptr)
    
          m_escapeNumBins = new uint16_t[m_symbolSize];
    
        initTBCTable(bitDepthY);
        if (m_indexError[0] == nullptr)
    
          for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
          {
            m_indexError[i] = new double[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
          }
        }
        if (m_minErrorIndexMap == nullptr)
        {
          m_minErrorIndexMap = new uint8_t[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
        }
        if (m_statePtRDOQ[0] == nullptr)
        {
          for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
          {
            m_statePtRDOQ[i] = new uint8_t[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
          }
    
    #if INTRA_TRANS_ENC_OPT
      m_skipTimdLfnstMtsPass = false;
    #endif
    
    #if JVET_AH0076_OBIC
      m_skipObicLfnstMtsPass = false;
      m_skipDimdLfnstMtsPass = false;
    #endif
    
    }
    
    
    //////////////////////////////////////////////////////////////////////////
    // INTRA PREDICTION
    //////////////////////////////////////////////////////////////////////////
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
    static constexpr double COST_UNKNOWN = -65536.0;
    
    
    double IntraSearch::findInterCUCost( CodingUnit &cu )
    {
      if( cu.isConsIntra() && !cu.slice->isIntra() )
      {
        //search corresponding inter CU cost
        for( int i = 0; i < m_numCuInSCIPU; i++ )
        {
          if( cu.lumaPos() == m_cuAreaInSCIPU[i].pos() && cu.lumaSize() == m_cuAreaInSCIPU[i].size() )
          {
            return m_cuCostInSCIPU[i];
          }
        }
      }
      return COST_UNKNOWN;
    }
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
    #if JVET_W0103_INTRA_MTS
    bool IntraSearch::testISPforCurrCU(const CodingUnit &cu)
    {
      CodingStructure       &cs = *cu.cs;
      auto &pu = *cu.firstPU;
      const CompArea &area = pu.Y();
      PelBuf piOrg = cs.getOrgBuf(area);
    
      Pel* pOrg = piOrg.buf;
      int uiWidth = area.width;
      int uiHeight = area.height;
      int iStride = piOrg.stride;
      int Gsum = 0;
      int nPix = (uiWidth - 2) * (uiHeight - 2);
      for (int y = 1; y < (uiHeight - 1); y++)
      {
        for (int x = 1; x < (uiWidth - 1); x++)
        {
          const Pel *p = pOrg + y * iStride + x;
    
          int iDy = p[-iStride - 1] + 2 * p[-1] + p[iStride - 1] - p[-iStride + 1] - 2 * p[+1] - p[iStride + 1];
          int iDx = p[iStride - 1] + 2 * p[iStride] + p[iStride + 1] - p[-iStride - 1] - 2 * p[-iStride] - p[-iStride + 1];
    
          if (iDy == 0 && iDx == 0)
            continue;
    
          int iAmp = (int)(abs(iDx) + abs(iDy));
          Gsum += iAmp;
        }
      }
      Gsum = (Gsum + (nPix >> 1)) / nPix;
    
      bool testISP = true;
      CHECK(m_numModesISPRDO != -1, "m_numModesISPRDO!=-1");
    
      m_numModesISPRDO = (Gsum < 50 && uiWidth >= 16 && uiHeight >= 16) ? 1 : 2;
      return testISP;
    }
    #endif
    
    bool IntraSearch::estIntraPredLumaQT(CodingUnit &cu, Partitioner &partitioner, const double bestCostSoFar, bool mtsCheckRangeFlag, int mtsFirstCheckId, int mtsLastCheckId, bool moreProbMTSIdxFirst, CodingStructure* bestCS
    #if JVET_AG0136_INTRA_TMP_LIC
                                         , InterPrediction* pcInterPred 
    #endif
    )
    
    {
      CodingStructure       &cs            = *cu.cs;
      const SPS             &sps           = *cs.sps;
    
      const uint32_t             uiWidthBit    = floorLog2(partitioner.currArea().lwidth() );
      const uint32_t             uiHeightBit   =                   floorLog2(partitioner.currArea().lheight());
    
    
      // Lambda calculation at equivalent Qp of 4 is recommended because at that Qp, the quantization divisor is 1.
    
      const double sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda( ) * FRAC_BITS_SCALE;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStart           ( m_ctxCache, m_CABACEstimator->getCtx() );
      const TempCtx ctxStartMipFlag    ( m_ctxCache, SubCtx( Ctx::MipFlag,          m_CABACEstimator->getCtx() ) );
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if JVET_V0130_INTRA_TMP
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartTpmFlag    ( m_ctxCache, SubCtx(Ctx::TmpFlag, m_CABACEstimator->getCtx()));
    
    Keming Cao's avatar
    Keming Cao committed
    #endif
    
    #if JVET_AD0086_ENHANCED_INTRA_TMP
      const TempCtx ctxStartTpmIdx     ( m_ctxCache, SubCtx(Ctx::TmpIdx, m_CABACEstimator->getCtx()));
      const TempCtx ctxStartTpmFusionFlag(m_ctxCache, SubCtx(Ctx::TmpFusion, m_CABACEstimator->getCtx()));
    #endif
    
    
    #if JVET_AG0136_INTRA_TMP_LIC
      const TempCtx ctxStartTmpLicFlag   (m_ctxCache, SubCtx(Ctx::TmpLic, m_CABACEstimator->getCtx()));
      const TempCtx ctxStartTmpLicIdx    (m_ctxCache, SubCtx(Ctx::ItmpLicIndex, m_CABACEstimator->getCtx()));
    #endif
    
    Keming Cao's avatar
    Keming Cao committed
    #if JVET_W0123_TIMD_FUSION
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartTimdFlag   ( m_ctxCache, SubCtx( Ctx::TimdFlag,      m_CABACEstimator->getCtx() ) );
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
    fan wang's avatar
    fan wang committed
    #if JVET_AB0155_SGPM
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartSgpmFlag   ( m_ctxCache, SubCtx(Ctx::SgpmFlag, m_CABACEstimator->getCtx()));
    
    fan wang's avatar
    fan wang committed
    #endif
    
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartIspMode    ( m_ctxCache, SubCtx( Ctx::ISPMode,          m_CABACEstimator->getCtx() ) );
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if SECONDARY_MPM
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartMPMIdxFlag ( m_ctxCache, SubCtx(Ctx::IntraLumaMPMIdx, m_CABACEstimator->getCtx()));
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartPlanarFlag ( m_ctxCache, SubCtx( Ctx::IntraLumaPlanarFlag, m_CABACEstimator->getCtx() ) );
      const TempCtx ctxStartIntraMode  ( m_ctxCache, SubCtx(Ctx::IntraLumaMpmFlag, m_CABACEstimator->getCtx()));
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if SECONDARY_MPM
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartIntraMode2 ( m_ctxCache, SubCtx(Ctx::IntraLumaSecondMpmFlag, m_CABACEstimator->getCtx()));
    
    #if JVET_AD0085_MPM_SORTING
      const TempCtx ctxStartMpmIdx2    ( m_ctxCache, SubCtx(Ctx::IntraLumaSecondMpmIdx, m_CABACEstimator->getCtx()) );
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartMrlIdx     ( m_ctxCache, SubCtx( Ctx::MultiRefLineIdx,        m_CABACEstimator->getCtx() ) );
    
    #if JVET_AB0157_TMRL
    
    Vadim Seregin's avatar
    Vadim Seregin committed
      const TempCtx ctxStartTmrlDerive ( m_ctxCache, SubCtx(Ctx::TmrlDerive, m_CABACEstimator->getCtx()));
    
    #if JVET_AG0058_EIP
      const TempCtx ctxStartEip(m_ctxCache, SubCtx(Ctx::EipFlag, m_CABACEstimator->getCtx()));
    #endif
    
    
      // NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
      auto loadStartStates = [&]()
      {
        m_CABACEstimator->getCtx() = SubCtx(Ctx::MipFlag, ctxStartMipFlag);
    #if JVET_V0130_INTRA_TMP
        m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpFlag, ctxStartTpmFlag);
    #endif
    #if JVET_AD0086_ENHANCED_INTRA_TMP
        m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpIdx, ctxStartTpmIdx);
        m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpFusion, ctxStartTpmFusionFlag);
    #endif
    #if JVET_W0123_TIMD_FUSION
        m_CABACEstimator->getCtx() = SubCtx(Ctx::TimdFlag, ctxStartTimdFlag);
    #endif
    #if JVET_AB0155_SGPM
        m_CABACEstimator->getCtx() = SubCtx(Ctx::SgpmFlag, ctxStartSgpmFlag);
    #endif
    
        m_CABACEstimator->getCtx() = SubCtx(Ctx::ISPMode, ctxStartIspMode);
    #if SECONDARY_MPM
        m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMPMIdx, ctxStartMPMIdxFlag);
    #endif
        m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
        m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
    #if SECONDARY_MPM
        m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaSecondMpmFlag, ctxStartIntraMode2);
    #if JVET_AD0085_MPM_SORTING
        m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaSecondMpmIdx, ctxStartMpmIdx2);
    #endif
    #endif
        m_CABACEstimator->getCtx() = SubCtx(Ctx::MultiRefLineIdx, ctxStartMrlIdx);
    #if JVET_AB0157_TMRL
        m_CABACEstimator->getCtx() = SubCtx(Ctx::TmrlDerive, ctxStartTmrlDerive);
    
    #endif
    #if JVET_AG0058_EIP
        m_CABACEstimator->getCtx() = SubCtx(Ctx::EipFlag, ctxStartEip);
    
    #endif
    #if JVET_AG0136_INTRA_TMP_LIC
                          m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpLic, ctxStartTmpLicFlag);
                          m_CABACEstimator->getCtx() = SubCtx(Ctx::ItmpLicIndex, ctxStartTmpLicIdx);
    
    #if JVET_AH0103_LOW_DELAY_LFNST_NSPT
      bool spsIntraLfnstEnabled = ( ( cu.slice->getSliceType() == I_SLICE && cu.cs->sps->getUseIntraLFNSTISlice() ) ||
                                    ( cu.slice->getSliceType() != I_SLICE && cu.cs->sps->getUseIntraLFNSTPBSlice() ) );
    #endif
    
      // variables for saving fast intra modes scan results across multiple LFNST passes
    
    #if JVET_AH0103_LOW_DELAY_LFNST_NSPT
      bool LFNSTLoadFlag = spsIntraLfnstEnabled && cu.lfnstIdx != 0;
      bool LFNSTSaveFlag = spsIntraLfnstEnabled && cu.lfnstIdx == 0;
    #else
    
      bool LFNSTLoadFlag = sps.getUseLFNST() && cu.lfnstIdx != 0;
      bool LFNSTSaveFlag = sps.getUseLFNST() && cu.lfnstIdx == 0;
    
    
      LFNSTSaveFlag &= sps.getUseIntraMTS() ? cu.mtsFlag == 0 : true;
    
    fan wang's avatar
    fan wang committed
    #if JVET_AB0155_SGPM
    
    fan wang's avatar
    fan wang committed
      bool SGPMSaveFlag = (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
    
    fan wang's avatar
    fan wang committed
    #endif
    
    #if JVET_AH0076_OBIC
      bool testDimd = isLuma(partitioner.chType) && cu.slice->getSPS()->getUseDimd();
      bool testObic = testDimd && (PU::isObicAvail(*cu.firstPU) && cu.obicMode[0] >= 0);
      bool obicSaveFlag = testObic && (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
      bool dimdSaveFlag = testDimd && (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
    #endif
    
      const uint32_t lfnstIdx = cu.lfnstIdx;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    
      double costInterCU = findInterCUCost( cu );
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
      const int width  = partitioner.currArea().lwidth();
      const int height = partitioner.currArea().lheight();
    
      // Marking MTS usage for faster MTS
      // 0: MTS is either not applicable for current CU (cuWidth > MTS_INTRA_MAX_CU_SIZE or cuHeight > MTS_INTRA_MAX_CU_SIZE), not active in the config file or the fast decision algorithm is not used in this case
      // 1: MTS fast algorithm can be applied for the current CU, and the DCT2 is being checked
      // 2: MTS is being checked for current CU. Stored results of DCT2 can be utilized for speedup
      uint8_t mtsUsageFlag = 0;
      const int maxSizeEMT = MTS_INTRA_MAX_CU_SIZE;
      if( width <= maxSizeEMT && height <= maxSizeEMT && sps.getUseIntraMTS() )
      {
    
    #if JVET_AH0103_LOW_DELAY_LFNST_NSPT
        mtsUsageFlag = ( spsIntraLfnstEnabled && cu.mtsFlag == 1 ) ? 2 : 1;
    #else
    
        mtsUsageFlag = ( sps.getUseLFNST() && cu.mtsFlag == 1 ) ? 2 : 1;
    
      }
    
      if( width * height < 64 && !m_pcEncCfg->getUseFastLFNST() )
      {
        mtsUsageFlag = 0;
      }
    
    #if JVET_W0103_INTRA_MTS
      if (!cu.mtsFlag && !cu.lfnstIdx)
      {
        m_globalBestCostStore = MAX_DOUBLE;
        m_globalBestCostValid = false;
        if (bestCS->getCU(partitioner.chType) != NULL && bestCS->getCU(partitioner.chType)->predMode != MODE_INTRA && bestCostSoFar != MAX_DOUBLE)
        {
          m_globalBestCostStore = bestCostSoFar;
          m_globalBestCostValid = true;
        }
    
    #if JVET_Y0142_ADAPT_INTRA_MTS
        m_modesForMTS.clear();
        m_modesCoeffAbsSumDCT2.clear();
    
    #if JVET_AE0169_BIPREDICTIVE_IBC
        m_bestIntraSADHADCost = MAX_DOUBLE;
    #endif
    
    #if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
        m_bestIntraSADCost = MAX_DOUBLE;
    
      const bool colorTransformIsEnabled = sps.getUseColorTrans() && !CS::isDualITree(cs);
      const bool isFirstColorSpace       = colorTransformIsEnabled && ((m_pcEncCfg->getRGBFormatFlag() && cu.colorTransform) || (!m_pcEncCfg->getRGBFormatFlag() && !cu.colorTransform));
      const bool isSecondColorSpace      = colorTransformIsEnabled && ((m_pcEncCfg->getRGBFormatFlag() && !cu.colorTransform) || (!m_pcEncCfg->getRGBFormatFlag() && cu.colorTransform));
    
      bool ispCanBeUsed   = sps.getUseISP() && cu.mtsFlag == 0 && cu.lfnstIdx == 0 && CU::canUseISP(width, height, cu.cs->sps->getMaxTbSize());
      bool saveDataForISP = ispCanBeUsed && (!colorTransformIsEnabled || isFirstColorSpace);
      bool testISP        = ispCanBeUsed && (!colorTransformIsEnabled || !cu.colorTransform);
    
    fan wang's avatar
    fan wang committed
    
    #if JVET_AB0155_SGPM
      const bool sgpmAllowed = sps.getUseSgpm() && isLuma(partitioner.chType);
      bool testSgpm = sgpmAllowed && cu.lwidth() >= GEO_MIN_CU_SIZE_EX && cu.lheight() >= GEO_MIN_CU_SIZE_EX
                      && cu.lwidth() <= GEO_MAX_CU_SIZE_EX && cu.lheight() <= GEO_MAX_CU_SIZE_EX
                      && cu.lwidth() < 8 * cu.lheight() && cu.lheight() < 8 * cu.lwidth() && cu.lx() && cu.ly()
                      && cu.lwidth() * cu.lheight() >= SGPM_MIN_PIX;
    #endif
    
    #if JVET_AG0058_EIP
      bool testEip = isLuma(partitioner.chType) && sps.getUseEip() && (getAllowedEip(cu, COMPONENT_Y) || getAllowedEipMerge(cu, COMPONENT_Y));
      bool eipSaveFlag = (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
    #endif
    
    #if JVET_W0103_INTRA_MTS 
      if (testISP && m_pcEncCfg->getUseFastISP())
      {
        m_numModesISPRDO = -1;
        testISP &= testISPforCurrCU(cu);
      }
    #endif
    
      if ( saveDataForISP )
    
        //reset the intra modes lists variables
        m_ispCandListHor.clear();
        m_ispCandListVer.clear();
    
        //reset the variables used for the tests
    
        m_regIntraRDListWithCosts.clear();
    
        int numTotalPartsHor = (int)width  >> floorLog2(CU::getISPSplitDim(width, height, TU_1D_VERT_SPLIT));
        int numTotalPartsVer = (int)height >> floorLog2(CU::getISPSplitDim(width, height, TU_1D_HORZ_SPLIT));
    
        m_ispTestedModes[0].init( numTotalPartsHor, numTotalPartsVer );
        //the total number of subpartitions is modified to take into account the cases where LFNST cannot be combined with ISP due to size restrictions
    
    #if JVET_AH0103_LOW_DELAY_LFNST_NSPT
        numTotalPartsHor = spsIntraLfnstEnabled && CU::canUseLfnstWithISP( cu.Y(), HOR_INTRA_SUBPARTITIONS ) ? numTotalPartsHor : 0;
        numTotalPartsVer = spsIntraLfnstEnabled && CU::canUseLfnstWithISP( cu.Y(), VER_INTRA_SUBPARTITIONS ) ? numTotalPartsVer : 0;
    #else
    
        numTotalPartsHor = sps.getUseLFNST() && CU::canUseLfnstWithISP(cu.Y(), HOR_INTRA_SUBPARTITIONS) ? numTotalPartsHor : 0;
        numTotalPartsVer = sps.getUseLFNST() && CU::canUseLfnstWithISP(cu.Y(), VER_INTRA_SUBPARTITIONS) ? numTotalPartsVer : 0;
    
        for (int j = 1; j < NUM_LFNST_NUM_PER_SET; j++)
        {
          m_ispTestedModes[j].init(numTotalPartsHor, numTotalPartsVer);
        }
    
    #if INTRA_TRANS_ENC_OPT
      double regAngCost = MAX_DOUBLE;
      bool setSkipTimdControl = (m_pcEncCfg->getIntraPeriod() == 1) && !cu.lfnstIdx && !cu.mtsFlag;
      double timdAngCost = MAX_DOUBLE;
    
    #endif
    #if JVET_AH0076_OBIC
      double obicAngCost = MAX_DOUBLE, dimdAngCost = MAX_DOUBLE;
      bool setSkipDimdControl = (m_pcEncCfg->getIntraPeriod() == 1) && !cu.lfnstIdx && !cu.mtsFlag;
    
      const bool testBDPCM = sps.getBDPCMEnabledFlag() && CU::bdpcmAllowed(cu, ComponentID(partitioner.chType)) && cu.mtsFlag == 0 && cu.lfnstIdx == 0;
    
      static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiHadModeList;
    
      static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostList;
    
      static_vector<double, FAST_UDI_MAX_RDMODE_NUM> CandHadList;
    
      auto &pu = *cu.firstPU;
    
      bool validReturn = false;
    
    #if JVET_AH0200_INTRA_TMP_BV_REORDER
        double tmpBestSatdCost = MAX_DOUBLE;
        CodedCUInfo &relatedCU = ((EncModeCtrlMTnoRQT *) m_modeCtrl)->getBlkInfo(partitioner.currArea());
        bool isTmpModeTestd = false;
    #endif	
    
        CHECK(pu.cu != &cu, "PU is not contained in the CU");
    
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    
    
        //===== determine set of modes to be tested (using prediction signal only) =====
        int numModesAvailable = NUM_LUMA_MODE; // total number of Intra modes
    
        const bool fastMip    = sps.getUseMIP() && m_pcEncCfg->getUseFastMIP();
    
        const bool mipAllowed = sps.getUseMIP() && isLuma(partitioner.chType) && ((cu.lfnstIdx == 0) || allowLfnstWithMip(cu.firstPU->lumaSize()));
    
        const bool testMip = mipAllowed && !(cu.lwidth() > (8 * cu.lheight()) || cu.lheight() > (8 * cu.lwidth()));
    
        const bool supportedMipBlkSize = pu.lwidth() <= MIP_MAX_WIDTH && pu.lheight() <= MIP_MAX_HEIGHT;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if JVET_V0130_INTRA_TMP
        const bool tpmAllowed = sps.getUseIntraTMP() && isLuma(partitioner.chType) && ((cu.lfnstIdx == 0) || allowLfnstWithTmp());
        const bool testTpm = tpmAllowed && (cu.lwidth() <= sps.getIntraTMPMaxSize() && cu.lheight() <= sps.getIntraTMPMaxSize());
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
    
        static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeList;
    
    Karsten Suehring's avatar
    Karsten Suehring committed
        numModesForFullRD = g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];
    
    #if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
        if (m_pcEncCfg->getIBCFastMethod() & IBC_FAST_METHOD_NONSCC)
        {
          numModesForFullRD = (numModesForFullRD > 1) ? (numModesForFullRD - 1) : numModesForFullRD;
        }
    #endif
    
    #if INTRA_FULL_SEARCH
        numModesForFullRD = numModesAvailable;
    #endif
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #if ENABLE_DIMD
        bool bestDimdMode = false;
    
    Keming Cao's avatar
    Keming Cao committed
    #endif
    
    #if JVET_AH0076_OBIC
        bool bestObicMode = false;
        int bestMipDimd = 0;
    #endif
    
    Keming Cao's avatar
    Keming Cao committed
    #if JVET_W0123_TIMD_FUSION
        bool bestTimdMode = false;
    
    Vadim Seregin's avatar
    Vadim Seregin committed
    #endif
    
    #if JVET_AC0105_DIRECTIONAL_PLANAR
        uint8_t bestPlMode = 0;
    #endif
    
    fan wang's avatar
    fan wang committed
    #if JVET_AB0155_SGPM
        bool bestSgpmMode = false;
        const CompArea &area = pu.Y();
        CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
    #endif
    
    #if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST 
        int intraTmpDimdMode = 0;
    #endif 
    
        if (isSecondColorSpace)
        {
          uiRdModeList.clear();
          if (m_numSavedRdModeFirstColorSpace[m_savedRdModeIdx] > 0)
          {
            for (int i = 0; i < m_numSavedRdModeFirstColorSpace[m_savedRdModeIdx]; i++)
            {
              uiRdModeList.push_back(m_savedRdModeFirstColorSpace[m_savedRdModeIdx][i]);
            }
          }
    
          if (mtsUsageFlag != 2)
    
    fan wang's avatar
    fan wang committed
    #if JVET_AB0155_SGPM
    
    #if JVET_AH0076_OBIC
            if ((testSgpm && SGPMSaveFlag) || obicSaveFlag || dimdSaveFlag)
    #else
    
    fan wang's avatar
    fan wang committed
            if (testSgpm && SGPMSaveFlag)
    
    fan wang's avatar
    fan wang committed
            {
    
    #if JVET_AG0152_SGPM_ITMP_IBC
              for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
    #else
    
    fan wang's avatar
    fan wang committed
              for (int i = 0; i < NUM_LUMA_MODE; i++)
    
    fan wang's avatar
    fan wang committed
              {
                m_intraModeReady[i] = 0;
              }
            }
    #endif
    
            // this should always be true
            CHECK(!pu.Y().valid(), "PU is not valid");
    
    #if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
    
    #if JVET_AH0065_RELAX_LINE_BUFFER
            bool isFirstLineOfCtu = pu.block(COMPONENT_Y).y == 0;
    #else
    
            bool isFirstLineOfCtu     = (((pu.block(COMPONENT_Y).y) & ((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);
    
    #if JVET_Y0116_EXTENDED_MRL_LIST
            int  numOfPassesExtendRef = MRL_NUM_REF_LINES;
            if (!sps.getUseMRL() || isFirstLineOfCtu) 
            {
              numOfPassesExtendRef = 1;
            }
            else
            {