IntraSearch.cpp

/* The copyright in this software is being made available under the BSD
 * License, included below. This software may be subject to other third party
 * and contributor rights, including patent rights, and no such rights are
 * granted under this license.
 *
 * Copyright (c) 2010-2023, ITU/ISO/IEC
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *  * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
 *    be used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

/** \file     EncSearch.cpp
 *  \brief    encoder intra search class
 */

#include "IntraSearch.h"

#include "EncModeCtrl.h"

#include "CommonLib/CommonDef.h"
#include "CommonLib/Rom.h"
#include "CommonLib/Picture.h"
#include "CommonLib/UnitTools.h"
#if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER
#include "CommonLib/BilateralFilter.h"
#endif

#include "CommonLib/dtrace_next.h"
#include "CommonLib/dtrace_buffer.h"
#if JVET_AJ0249_NEURAL_NETWORK_BASED
#include "CommonLib/IntraPredictionNN.h"
#endif

#include <math.h>
#include <limits>
 //! \ingroup EncoderLib
 //! \{
#define PLTCtx(c) SubCtx( Ctx::Palette, c )
IntraSearch::IntraSearch()
  : m_pSplitCS      (nullptr)
  , m_pFullCS       (nullptr)
  , m_pBestCS       (nullptr)
  , m_pcEncCfg      (nullptr)
#if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER
  , m_bilateralFilter(nullptr)
#endif
  , m_pcTrQuant     (nullptr)
  , m_pcRdCost      (nullptr)
  , m_pcReshape     (nullptr)
  , m_CABACEstimator(nullptr)
  , m_ctxCache      (nullptr)
  , m_isInitialized (false)
{
  for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
  {
    m_pSharedPredTransformSkip[ch] = nullptr;
  }
#if JVET_AB0155_SGPM
#if JVET_AG0152_SGPM_ITMP_IBC
  for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
  for (int i = 0; i < NUM_LUMA_MODE; i++)
#endif
  {
    m_intraPredBuf[i] = nullptr;
  }
  for (int i = 0; i < SGPM_NUM; i++)
  {
    m_sgpmPredBuf[i] = nullptr;
  }
#endif
#if JVET_AH0209_PDP
  for (int i = 0; i < NUM_LUMA_MODE; i++)
  {
    m_pdpIntraPredBuf[i] = nullptr;
  }
#endif
#if JVET_AG0058_EIP
  for (int i = 0; i < NUM_DERIVED_EIP; i++)
  {
    m_eipPredBuf[i] = nullptr;
  }
  for(int i = 0; i < MAX_MERGE_EIP;i++)
  {
    m_eipMergePredBuf[i] = nullptr;
  }
#endif
#if JVET_AJ0061_TIMD_MERGE
  for (int i = 0; i < NumTimdMode ; i++)
  {
    m_timdPredBuf[i] = nullptr;
  }
#endif
#if JVET_AH0076_OBIC
  m_dimdPredBuf = nullptr;
  m_obicPredBuf = nullptr;
#endif
#if !JVET_AJ0237_INTERNAL_12BIT
  m_truncBinBits = nullptr;
  m_escapeNumBins = nullptr;
#endif
  m_minErrorIndexMap = nullptr;
  for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
  {
    m_indexError[i] = nullptr;
  }
  for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
  {
    m_statePtRDOQ[i] = nullptr;
  }

  ::memset( m_indexMapRDOQ, 0, sizeof( m_indexMapRDOQ ) );
  ::memset( m_runMapRDOQ, false, sizeof( m_runMapRDOQ ) );
  ::memset( m_prevRunTypeRDOQ, false, sizeof( m_prevRunTypeRDOQ ) );
  ::memset( m_prevRunPosRDOQ, 0, sizeof( m_prevRunPosRDOQ ) );
  ::memset( m_stateCostRDOQ, 0, sizeof( m_stateCostRDOQ ) );
}


void IntraSearch::destroy()
{
  CHECK( !m_isInitialized, "Not initialized" );

  if( m_pcEncCfg )
  {
    const uint32_t uiNumLayersToAllocateSplit = 1;
    const uint32_t uiNumLayersToAllocateFull  = 1;
    const int uiNumSaveLayersToAllocate = 2;

    for( uint32_t layer = 0; layer < uiNumSaveLayersToAllocate; layer++ )
    {
      m_pSaveCS[layer]->destroy();
      delete m_pSaveCS[layer];
    }

    uint32_t numWidths  = gp_sizeIdxInfo->numWidths();
    uint32_t numHeights = gp_sizeIdxInfo->numHeights();

    for( uint32_t width = 0; width < numWidths; width++ )
    {
      for( uint32_t height = 0; height < numHeights; height++ )
      {
        if( gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( width ) ) && gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( height ) )
          && gp_sizeIdxInfo->sizeFrom(width) <= m_pcEncCfg->getMaxCUWidth() && gp_sizeIdxInfo->sizeFrom(height) <= m_pcEncCfg->getMaxCUHeight())
        {
          for( uint32_t layer = 0; layer < uiNumLayersToAllocateSplit; layer++ )
          {
            m_pSplitCS[width][height][layer]->destroy();

            delete m_pSplitCS[width][height][layer];
          }

          for( uint32_t layer = 0; layer < uiNumLayersToAllocateFull; layer++ )
          {
            m_pFullCS[width][height][layer]->destroy();

            delete m_pFullCS[width][height][layer];
          }

          delete[] m_pSplitCS[width][height];
          delete[] m_pFullCS [width][height];

          m_pBestCS[width][height]->destroy();
          m_pTempCS[width][height]->destroy();

          delete m_pTempCS[width][height];
          delete m_pBestCS[width][height];
        }
      }

      delete[] m_pSplitCS[width];
      delete[] m_pFullCS [width];

      delete[] m_pTempCS[width];
      delete[] m_pBestCS[width];
    }

    delete[] m_pSplitCS;
    delete[] m_pFullCS;

    delete[] m_pBestCS;
    delete[] m_pTempCS;

    delete[] m_pSaveCS;
  }

  m_pSplitCS = m_pFullCS = nullptr;

  m_pBestCS = m_pTempCS = nullptr;

  m_pSaveCS = nullptr;

  for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
  {
    delete[] m_pSharedPredTransformSkip[ch];
    m_pSharedPredTransformSkip[ch] = nullptr;
  }

#if JVET_AJ0081_CHROMA_TMRL
  for (uint32_t i = 0; i < CHROMA_TMRL_LIST_SIZE; i++)
  {
    m_chromaMrlStorage[i].destroy();
  }
#endif

#if JVET_AC0119_LM_CHROMA_FUSION
  for (uint32_t i = 0; i < 2; i++)
  {
    m_predStorage[i].destroy();
  }
  for (uint32_t i = 0; i < 6; i++)
  {
    m_fusionStorage[i].destroy();
  }
#endif

#if JVET_AD0120_LBCCP
  for (uint32_t i = 0; i < LBCCP_FILTER_MMLMNUM; i++)
  {
    m_lmPredFiltStorage[i].destroy();
  }
#endif

#if JVET_AG0059_CCP_MERGE_ENHANCEMENT
  for (uint32_t i = 0; i < 2; i++)
  {
    m_predCCPFusionStorage[i].destroy();
  }
#endif

#if JVET_AB0143_CCCM_TS
#if JVET_AD0202_CCCM_MDF
  for (uint32_t cccmIdx = 0; cccmIdx < TOTAL_NUM_CCCM_MODES; cccmIdx++)
#else
  for (uint32_t cccmIdx = 0; cccmIdx < CCCM_NUM_MODES; cccmIdx++)
#endif
  {
#if JVET_AC0147_CCCM_NO_SUBSAMPLING
    m_cccmStorage[0][cccmIdx].destroy();
    m_cccmStorage[1][cccmIdx].destroy();
#else
    m_cccmStorage[cccmIdx].destroy();
#endif
  }
#endif
#if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
  m_ddCcpStorage.destroy();
  for (int i = 0; i < 2; i++)
  {
    m_ddCcpFusionStorage[i].destroy();
  }
#endif

  m_tmpStorageLCU.destroy();
  m_colorTransResiBuf.destroy();
  
#if JVET_AB0155_SGPM
#if JVET_AG0152_SGPM_ITMP_IBC
  for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
  for (int i = 0; i < NUM_LUMA_MODE; i++)
#endif
  {
    delete[] m_intraPredBuf[i];
    m_intraPredBuf[i] = nullptr;
  }

  for (int i = 0; i < SGPM_NUM; i++)
  {
    delete[] m_sgpmPredBuf[i];
    m_sgpmPredBuf[i] = nullptr;
  }
#endif
#if JVET_AH0209_PDP
  for (int i = 0; i < NUM_LUMA_MODE; i++)
  {
    if (m_pdpIntraPredBuf[i])
    {
      delete[] m_pdpIntraPredBuf[i];
      m_pdpIntraPredBuf[i] = nullptr;
    }
  }
#endif
#if JVET_AG0058_EIP
  for (int i = 0; i < NUM_DERIVED_EIP; i++)
  {
    delete[] m_eipPredBuf[i];
    m_eipPredBuf[i] = nullptr;
  }
  for(int i = 0; i < MAX_MERGE_EIP;i++)
  {
    delete[] m_eipMergePredBuf[i];
    m_eipMergePredBuf[i] = nullptr;
  }
#endif
#if JVET_AJ0061_TIMD_MERGE
  for (int i = 0; i < NumTimdMode ; i++)
  {
    delete[] m_timdPredBuf[i];
    m_timdPredBuf[i] = nullptr;
  }
#endif
#if JVET_AH0076_OBIC
  delete[] m_dimdPredBuf;
  m_dimdPredBuf = nullptr;
  delete[] m_obicPredBuf;
  m_obicPredBuf = nullptr;
#endif
  m_isInitialized = false;
#if !JVET_AJ0237_INTERNAL_12BIT
  if (m_truncBinBits != nullptr)
  {
    for (unsigned i = 0; i < m_symbolSize; i++)
    {
      delete[] m_truncBinBits[i];
      m_truncBinBits[i] = nullptr;
    }
    delete[] m_truncBinBits;
    m_truncBinBits = nullptr;
  }
  if (m_escapeNumBins != nullptr)
  {
    delete[] m_escapeNumBins;
    m_escapeNumBins = nullptr;
  }
#endif
  if (m_indexError[0] != nullptr)
  {
    for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
    {
      delete[] m_indexError[i];
      m_indexError[i] = nullptr;
    }
  }
  if (m_minErrorIndexMap != nullptr)
  {
    delete[] m_minErrorIndexMap;
    m_minErrorIndexMap = nullptr;
  }
  if (m_statePtRDOQ[0] != nullptr)
  {
    for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
    {
      delete[] m_statePtRDOQ[i];
      m_statePtRDOQ[i] = nullptr;
    }
  }
}

IntraSearch::~IntraSearch()
{
  if( m_isInitialized )
  {
    destroy();
  }
}
#if JVET_AJ0249_NEURAL_NETWORK_BASED
void IntraSearch::resetIndicesRepresentationPnnMemories()
{
  for (auto it{m_indicesRepresentationPnn.begin()}; it != m_indicesRepresentationPnn.end(); it++)
  {
    std::fill(it->begin(), it->end(), MAX_INT);
  }
}
#endif

void IntraSearch::init( EncCfg*        pcEncCfg,
#if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER
                       BilateralFilter* bilateralFilter,
#endif
                        TrQuant*       pcTrQuant,
                        RdCost*        pcRdCost,
                        CABACWriter*   CABACEstimator,
                        CtxCache*      ctxCache,
                        const uint32_t     maxCUWidth,
                        const uint32_t     maxCUHeight,
                        const uint32_t     maxTotalCUDepth
                       , EncReshape*   pcReshape
                       , const unsigned bitDepthY
)
{
  CHECK(m_isInitialized, "Already initialized");
  m_pcEncCfg                     = pcEncCfg;
#if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER
  m_bilateralFilter              = bilateralFilter;
#endif
  m_pcTrQuant                    = pcTrQuant;
  m_pcRdCost                     = pcRdCost;
  m_CABACEstimator               = CABACEstimator;
  m_ctxCache                     = ctxCache;
  m_pcReshape                    = pcReshape;

  const ChromaFormat cform = pcEncCfg->getChromaFormatIdc();

#if JVET_AJ0249_NEURAL_NETWORK_BASED
  IntraPrediction::init(cform, pcEncCfg->getBitDepth(CHANNEL_TYPE_LUMA), pcEncCfg->getNnipMode());
#else
  IntraPrediction::init( cform, pcEncCfg->getBitDepth( CHANNEL_TYPE_LUMA ) );
#endif
  m_tmpStorageLCU.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
  m_colorTransResiBuf.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));

#if JVET_AJ0081_CHROMA_TMRL
  for (uint32_t i = 0; i < CHROMA_TMRL_LIST_SIZE; i++)
  {
    m_chromaMrlStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
  }
#endif

#if JVET_AC0119_LM_CHROMA_FUSION
  for (uint32_t i = 0; i < 2; i++)
  {
    m_predStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
  }
  for (uint32_t i = 0; i < 6; i++)
  {
    m_fusionStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
  }
#endif

#if JVET_AD0120_LBCCP
  for (uint32_t i = 0; i < LBCCP_FILTER_MMLMNUM; i++)
  {
    m_lmPredFiltStorage[i].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
  }
#endif

#if JVET_AG0059_CCP_MERGE_ENHANCEMENT
  for (uint32_t i = 0; i < 2; i++)
  {
    m_predCCPFusionStorage[i].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
  }
#endif

#if JVET_AB0143_CCCM_TS
#if JVET_AD0202_CCCM_MDF
  for (uint32_t cccmIdx = 0; cccmIdx < TOTAL_NUM_CCCM_MODES; cccmIdx++)
#else
  for (uint32_t cccmIdx = 0; cccmIdx < CCCM_NUM_MODES; cccmIdx++)
#endif
  {
#if JVET_AC0147_CCCM_NO_SUBSAMPLING
    m_cccmStorage[0][cccmIdx].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
    m_cccmStorage[1][cccmIdx].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
#else
    m_cccmStorage[cccmIdx].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
#endif
  }
#endif
#if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
  m_ddCcpStorage.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
  for (int i = 0; i < 2; i++)
  {
    m_ddCcpFusionStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
  }
#endif
#if JVET_AB0155_SGPM || JVET_AH0076_OBIC
#if JVET_AG0152_SGPM_ITMP_IBC
  for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
  for (int i = 0; i < NUM_LUMA_MODE; i++)
#endif
  {
#if JVET_AH0076_OBIC
    m_intraPredBuf[i] = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
#else
    m_intraPredBuf[i] = new Pel[GEO_MAX_CU_SIZE_EX * GEO_MAX_CU_SIZE_EX];
#endif
  }
  for (int i = 0; i < SGPM_NUM; i++)
  {
    m_sgpmPredBuf[i] = new Pel[GEO_MAX_CU_SIZE_EX * GEO_MAX_CU_SIZE_EX];
  }
#endif
#if JVET_AH0209_PDP
  for (int i = 0; i < NUM_LUMA_MODE; i++)
  {
    if( i>2 && i%2 )
    {
      continue;
    }
    m_pdpIntraPredBuf[i] = new Pel[32*32]; //maxsize 32 where pdp is applied.
  }
#endif
#if JVET_AG0058_EIP
  for (int i = 0; i < NUM_DERIVED_EIP; i++)
  {
    m_eipPredBuf[i] = new Pel[MAX_EIP_SIZE * MAX_EIP_SIZE];
  }
  for(int i = 0; i < MAX_MERGE_EIP;i++)
  {
    m_eipMergePredBuf[i] = new Pel[MAX_EIP_SIZE * MAX_EIP_SIZE];
  }
#endif
#if JVET_AJ0061_TIMD_MERGE
  for (int i = 0; i < NumTimdMode; i++)
  {
    m_timdPredBuf[i] = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
  }
#endif
#if JVET_AH0076_OBIC
  m_dimdPredBuf = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
  m_obicPredBuf = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
#endif
  for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
  {
    m_pSharedPredTransformSkip[ch] = new Pel[maxCUWidth * maxCUHeight];
  }

  uint32_t numWidths  = gp_sizeIdxInfo->numWidths();
  uint32_t numHeights = gp_sizeIdxInfo->numHeights();

  const uint32_t uiNumLayersToAllocateSplit = 1;
  const uint32_t uiNumLayersToAllocateFull  = 1;

  m_pBestCS = new CodingStructure**[numWidths];
  m_pTempCS = new CodingStructure**[numWidths];

  m_pFullCS  = new CodingStructure***[numWidths];
  m_pSplitCS = new CodingStructure***[numWidths];

  for( uint32_t width = 0; width < numWidths; width++ )
  {
    m_pBestCS[width] = new CodingStructure*[numHeights];
    m_pTempCS[width] = new CodingStructure*[numHeights];

    m_pFullCS [width] = new CodingStructure**[numHeights];
    m_pSplitCS[width] = new CodingStructure**[numHeights];

    for( uint32_t height = 0; height < numHeights; height++ )
    {
      if(  gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( width ) ) && gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( height ) )
        && gp_sizeIdxInfo->sizeFrom(width) <= maxCUWidth && gp_sizeIdxInfo->sizeFrom(height) <= maxCUHeight)
      {
        m_pBestCS[width][height] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
        m_pTempCS[width][height] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );

#if JVET_Z0118_GDR
        m_pBestCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
        m_pTempCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
#else
        m_pBestCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
        m_pTempCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
#endif

        m_pFullCS [width][height] = new CodingStructure*[uiNumLayersToAllocateFull];
        m_pSplitCS[width][height] = new CodingStructure*[uiNumLayersToAllocateSplit];

        for( uint32_t layer = 0; layer < uiNumLayersToAllocateFull; layer++ )
        {
          m_pFullCS [width][height][layer] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );

#if JVET_Z0118_GDR
          m_pFullCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
#else
          m_pFullCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
#endif
        }

        for( uint32_t layer = 0; layer < uiNumLayersToAllocateSplit; layer++ )
        {
          m_pSplitCS[width][height][layer] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
#if JVET_Z0118_GDR
          m_pSplitCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
#else
          m_pSplitCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
#endif
        }
      }
      else
      {
        m_pBestCS[width][height] = nullptr;
        m_pTempCS[width][height] = nullptr;

        m_pFullCS [width][height] = nullptr;
        m_pSplitCS[width][height] = nullptr;
      }
    }
  }

  const int uiNumSaveLayersToAllocate = 2;

  m_pSaveCS = new CodingStructure*[uiNumSaveLayersToAllocate];

  for( uint32_t depth = 0; depth < uiNumSaveLayersToAllocate; depth++ )
  {
    m_pSaveCS[depth] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
#if JVET_Z0118_GDR
    m_pSaveCS[depth]->create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
#else
    m_pSaveCS[depth]->create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)), false, (bool)pcEncCfg->getPLTMode());
#endif
  }

  m_isInitialized = true;
  if (pcEncCfg->getPLTMode())
  {
#if !JVET_AJ0237_INTERNAL_12BIT
    m_symbolSize = (1 << bitDepthY); // pixel values are within [0, SymbolSize-1] with size SymbolSize
    if (m_truncBinBits == nullptr)
    {
      m_truncBinBits = new uint16_t*[m_symbolSize];
      for (unsigned i = 0; i < m_symbolSize; i++)
      {
        m_truncBinBits[i] = new uint16_t[m_symbolSize + 1];
      }
    }
    if (m_escapeNumBins == nullptr)
    {
      m_escapeNumBins = new uint16_t[m_symbolSize];
    }
    initTBCTable(bitDepthY);
#endif
    if (m_indexError[0] == nullptr)
    {
      for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
      {
        m_indexError[i] = new double[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
      }
    }
    if (m_minErrorIndexMap == nullptr)
    {
      m_minErrorIndexMap = new uint8_t[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
    }
    if (m_statePtRDOQ[0] == nullptr)
    {
      for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
      {
        m_statePtRDOQ[i] = new uint8_t[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
      }
    }
  }
#if INTRA_TRANS_ENC_OPT
  m_skipTimdLfnstMtsPass = false;
#endif
#if JVET_AJ0112_REGRESSION_SGPM
  m_skipSgpmLfnstMtsPass = false;
#endif
#if JVET_AJ0061_TIMD_MERGE
  m_skipTimdMrgLfnstMtsPass = false;
  m_skipObicMode            = false;
  m_skipDimdMode            = false;
  m_satdCostOBIC            = MAX_UINT64;
  m_satdCostDIMD            = MAX_UINT64;
  for (int i = 0; i < NumTimdMode ; i++)
  {
    m_skipTimdMode[i] = false;
    m_satdCostTIMD[i][0] = MAX_UINT64;
    m_satdCostTIMD[i][1] = MAX_UINT64;
  }
#endif
#if JVET_AH0076_OBIC
  m_skipObicLfnstMtsPass = false;
  m_skipDimdLfnstMtsPass = false;
#endif
#if JVET_AJ0082_MM_EIP
  m_skipEipLfnstMtsPass = false;
#endif
#if JVET_AJ0249_NEURAL_NETWORK_BASED
  m_skipNnLfnstMtsPass = false;
#endif
}


//////////////////////////////////////////////////////////////////////////
// INTRA PREDICTION
//////////////////////////////////////////////////////////////////////////
#if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
static constexpr double COST_UNKNOWN = -65536.0;

double IntraSearch::findInterCUCost( CodingUnit &cu )
{
  if( cu.isConsIntra() && !cu.slice->isIntra() )
  {
    //search corresponding inter CU cost
    for( int i = 0; i < m_numCuInSCIPU; i++ )
    {
      if( cu.lumaPos() == m_cuAreaInSCIPU[i].pos() && cu.lumaSize() == m_cuAreaInSCIPU[i].size() )
      {
        return m_cuCostInSCIPU[i];
      }
    }
  }
  return COST_UNKNOWN;
}
#endif
#if JVET_W0103_INTRA_MTS
bool IntraSearch::testISPforCurrCU(const CodingUnit &cu)
{
  CodingStructure       &cs = *cu.cs;
  auto &pu = *cu.firstPU;
  const CompArea &area = pu.Y();
  PelBuf piOrg = cs.getOrgBuf(area);

  Pel* pOrg = piOrg.buf;
  int uiWidth = area.width;
  int uiHeight = area.height;
  int iStride = piOrg.stride;
  int Gsum = 0;
  int nPix = (uiWidth - 2) * (uiHeight - 2);
  for (int y = 1; y < (uiHeight - 1); y++)
  {
    for (int x = 1; x < (uiWidth - 1); x++)
    {
      const Pel *p = pOrg + y * iStride + x;

      int iDy = p[-iStride - 1] + 2 * p[-1] + p[iStride - 1] - p[-iStride + 1] - 2 * p[+1] - p[iStride + 1];
      int iDx = p[iStride - 1] + 2 * p[iStride] + p[iStride + 1] - p[-iStride - 1] - 2 * p[-iStride] - p[-iStride + 1];

      if (iDy == 0 && iDx == 0)
        continue;

      int iAmp = (int)(abs(iDx) + abs(iDy));
      Gsum += iAmp;
    }
  }
  Gsum = (Gsum + (nPix >> 1)) / nPix;

  bool testISP = true;
  CHECK(m_numModesISPRDO != -1, "m_numModesISPRDO!=-1");

  m_numModesISPRDO = (Gsum < 50 && uiWidth >= 16 && uiHeight >= 16) ? 1 : 2;
  return testISP;
}
#endif
bool IntraSearch::estIntraPredLumaQT(CodingUnit &cu, Partitioner &partitioner, const double bestCostSoFar, bool mtsCheckRangeFlag, int mtsFirstCheckId, int mtsLastCheckId, bool moreProbMTSIdxFirst, CodingStructure* bestCS
#if JVET_AG0136_INTRA_TMP_LIC
                                     , InterPrediction* pcInterPred 
#endif
)
{
  CodingStructure       &cs            = *cu.cs;
  const SPS             &sps           = *cs.sps;
  const uint32_t             uiWidthBit    = floorLog2(partitioner.currArea().lwidth() );
  const uint32_t             uiHeightBit   =                   floorLog2(partitioner.currArea().lheight());

  // Lambda calculation at equivalent Qp of 4 is recommended because at that Qp, the quantization divisor is 1.
  const double sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda( ) * FRAC_BITS_SCALE;

  //===== loop over partitions =====

  const TempCtx ctxStart           ( m_ctxCache, m_CABACEstimator->getCtx() );
  const TempCtx ctxStartMipFlag    ( m_ctxCache, SubCtx( Ctx::MipFlag,          m_CABACEstimator->getCtx() ) );
#if JVET_V0130_INTRA_TMP
  const TempCtx ctxStartTpmFlag    ( m_ctxCache, SubCtx(Ctx::TmpFlag, m_CABACEstimator->getCtx()));
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
  const TempCtx ctxStartTpmIdx     ( m_ctxCache, SubCtx(Ctx::TmpIdx, m_CABACEstimator->getCtx()));
  const TempCtx ctxStartTpmFusionFlag(m_ctxCache, SubCtx(Ctx::TmpFusion, m_CABACEstimator->getCtx()));
#endif
#if JVET_AJ0249_NEURAL_NETWORK_BASED
  const bool isNnIn = cu.slice->getPnnMode();
  const TempCtx ctxStartIntraNnFlag(m_ctxCache, SubCtx(Ctx::PnnLuminanceFlag, m_CABACEstimator->getCtx()));
#if ENABLE_DIMD
  const TempCtx ctxStartDimdFlag(m_ctxCache, SubCtx(Ctx::DimdFlag, m_CABACEstimator->getCtx()));
#if JVET_AH0076_OBIC
  const TempCtx ctxStartObicFlag(m_ctxCache, SubCtx(Ctx::obicFlag, m_CABACEstimator->getCtx()));
#endif
#endif
#endif

#if JVET_AG0136_INTRA_TMP_LIC
  const TempCtx ctxStartTmpLicFlag   (m_ctxCache, SubCtx(Ctx::TmpLic, m_CABACEstimator->getCtx()));
  const TempCtx ctxStartTmpLicIdx    (m_ctxCache, SubCtx(Ctx::ItmpLicIndex, m_CABACEstimator->getCtx()));
#endif
#if JVET_W0123_TIMD_FUSION
  const TempCtx ctxStartTimdFlag   ( m_ctxCache, SubCtx( Ctx::TimdFlag,      m_CABACEstimator->getCtx() ) );
#if JVET_AJ0061_TIMD_MERGE
  const TempCtx ctxStartTimdMrgFlag( m_ctxCache, SubCtx( Ctx::TimdMrgFlag,   m_CABACEstimator->getCtx() ) );
#endif
#endif
#if JVET_AB0155_SGPM
  const TempCtx ctxStartSgpmFlag   ( m_ctxCache, SubCtx(Ctx::SgpmFlag, m_CABACEstimator->getCtx()));
#endif

  const TempCtx ctxStartIspMode    ( m_ctxCache, SubCtx( Ctx::ISPMode,          m_CABACEstimator->getCtx() ) );
#if SECONDARY_MPM
  const TempCtx ctxStartMPMIdxFlag ( m_ctxCache, SubCtx(Ctx::IntraLumaMPMIdx, m_CABACEstimator->getCtx()));
#endif
  const TempCtx ctxStartPlanarFlag ( m_ctxCache, SubCtx( Ctx::IntraLumaPlanarFlag, m_CABACEstimator->getCtx() ) );
  const TempCtx ctxStartIntraMode  ( m_ctxCache, SubCtx(Ctx::IntraLumaMpmFlag, m_CABACEstimator->getCtx()));
#if SECONDARY_MPM
  const TempCtx ctxStartIntraMode2 ( m_ctxCache, SubCtx(Ctx::IntraLumaSecondMpmFlag, m_CABACEstimator->getCtx()));
#if JVET_AD0085_MPM_SORTING
  const TempCtx ctxStartMpmIdx2    ( m_ctxCache, SubCtx(Ctx::IntraLumaSecondMpmIdx, m_CABACEstimator->getCtx()) );
#endif
#endif
  const TempCtx ctxStartMrlIdx     ( m_ctxCache, SubCtx( Ctx::MultiRefLineIdx,        m_CABACEstimator->getCtx() ) );
#if JVET_AB0157_TMRL
  const TempCtx ctxStartTmrlDerive ( m_ctxCache, SubCtx(Ctx::TmrlDerive, m_CABACEstimator->getCtx()));
#endif
#if JVET_AG0058_EIP
  const TempCtx ctxStartEip(m_ctxCache, SubCtx(Ctx::EipFlag, m_CABACEstimator->getCtx()));
#endif

  // NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
  auto loadStartStates = [&]()
  {
    m_CABACEstimator->getCtx() = SubCtx(Ctx::MipFlag, ctxStartMipFlag);
#if JVET_V0130_INTRA_TMP
    m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpFlag, ctxStartTpmFlag);
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
    m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpIdx, ctxStartTpmIdx);
    m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpFusion, ctxStartTpmFusionFlag);
#endif
#if JVET_AJ0249_NEURAL_NETWORK_BASED
    if (isNnIn)
    {
      m_CABACEstimator->getCtx() = SubCtx(Ctx::PnnLuminanceFlag, ctxStartIntraNnFlag);
#if ENABLE_DIMD
      m_CABACEstimator->getCtx() = SubCtx(Ctx::DimdFlag, ctxStartDimdFlag);
#if JVET_AH0076_OBIC
      m_CABACEstimator->getCtx() = SubCtx(Ctx::obicFlag, ctxStartObicFlag);
#endif
#endif
    }
#endif
#if JVET_W0123_TIMD_FUSION
    m_CABACEstimator->getCtx() = SubCtx(Ctx::TimdFlag, ctxStartTimdFlag);
#if JVET_AJ0061_TIMD_MERGE
    m_CABACEstimator->getCtx() = SubCtx(Ctx::TimdMrgFlag, ctxStartTimdMrgFlag);
#endif
#endif
#if JVET_AB0155_SGPM
    m_CABACEstimator->getCtx() = SubCtx(Ctx::SgpmFlag, ctxStartSgpmFlag);
#endif

    m_CABACEstimator->getCtx() = SubCtx(Ctx::ISPMode, ctxStartIspMode);
#if SECONDARY_MPM
    m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMPMIdx, ctxStartMPMIdxFlag);
#endif
    m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
    m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
#if SECONDARY_MPM
    m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaSecondMpmFlag, ctxStartIntraMode2);
#if JVET_AD0085_MPM_SORTING
    m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaSecondMpmIdx, ctxStartMpmIdx2);
#endif
#endif
    m_CABACEstimator->getCtx() = SubCtx(Ctx::MultiRefLineIdx, ctxStartMrlIdx);
#if JVET_AB0157_TMRL
    m_CABACEstimator->getCtx() = SubCtx(Ctx::TmrlDerive, ctxStartTmrlDerive);
#endif
#if JVET_AG0058_EIP
    m_CABACEstimator->getCtx() = SubCtx(Ctx::EipFlag, ctxStartEip);
#endif
#if JVET_AG0136_INTRA_TMP_LIC
                      m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpLic, ctxStartTmpLicFlag);
                      m_CABACEstimator->getCtx() = SubCtx(Ctx::ItmpLicIndex, ctxStartTmpLicIdx);
#endif
  };

#if JVET_AJ0061_TIMD_MERGE
  bool isTimdValid = cu.slice->getSPS()->getUseTimd() && !(cu.lwidth() * cu.lheight() > 1024 && cu.slice->getSliceType() == I_SLICE);
  static_vector<ModeInfo, NumTimdMode> timdModes;
  static_vector<double,   NumTimdMode> timdCosts;
  static_vector<double,   NumTimdMode> timdSadCosts;
  int numTimdSatd = 0;
  timdModes.clear();
  timdCosts.clear();
  timdSadCosts.clear();
#endif
  CHECK( !cu.firstPU, "CU has no PUs" );
#if JVET_AH0103_LOW_DELAY_LFNST_NSPT
  bool spsIntraLfnstEnabled = ( ( cu.slice->getSliceType() == I_SLICE && cu.cs->sps->getUseIntraLFNSTISlice() ) ||
                                ( cu.slice->getSliceType() != I_SLICE && cu.cs->sps->getUseIntraLFNSTPBSlice() ) );
#endif
  // variables for saving fast intra modes scan results across multiple LFNST passes
#if JVET_AH0103_LOW_DELAY_LFNST_NSPT
  bool LFNSTLoadFlag = spsIntraLfnstEnabled && cu.lfnstIdx != 0;
  bool LFNSTSaveFlag = spsIntraLfnstEnabled && cu.lfnstIdx == 0;
#else
  bool LFNSTLoadFlag = sps.getUseLFNST() && cu.lfnstIdx != 0;
  bool LFNSTSaveFlag = sps.getUseLFNST() && cu.lfnstIdx == 0;
#endif

  LFNSTSaveFlag &= sps.getUseIntraMTS() ? cu.mtsFlag == 0 : true;
#if JVET_AB0155_SGPM
  bool SGPMSaveFlag = (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
#endif

#if JVET_AH0076_OBIC
  bool testDimd = isLuma(partitioner.chType) && cu.slice->getSPS()->getUseDimd();
  bool testObic = testDimd && (PU::isObicAvail(*cu.firstPU) && cu.obicMode[0] >= 0);
  bool obicSaveFlag = testObic && (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
  bool dimdSaveFlag = testDimd && (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
#endif
#if JVET_AH0209_PDP
  bool pdpSaveFlag = !cu.lfnstIdx && !cu.mtsFlag;
  if (pdpSaveFlag)
  {
    std::memset( m_pdpIntraPredReady, 0, sizeof( m_pdpIntraPredReady ) );
  }
#endif
  const uint32_t lfnstIdx = cu.lfnstIdx;
#if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
  double costInterCU = findInterCUCost( cu );
#endif
  const int width  = partitioner.currArea().lwidth();
  const int height = partitioner.currArea().lheight();

  // Marking MTS usage for faster MTS
  // 0: MTS is either not applicable for current CU (cuWidth > MTS_INTRA_MAX_CU_SIZE or cuHeight > MTS_INTRA_MAX_CU_SIZE), not active in the config file or the fast decision algorithm is not used in this case
  // 1: MTS fast algorithm can be applied for the current CU, and the DCT2 is being checked
  // 2: MTS is being checked for current CU. Stored results of DCT2 can be utilized for speedup
  uint8_t mtsUsageFlag = 0;
  const int maxSizeEMT = MTS_INTRA_MAX_CU_SIZE;
  if( width <= maxSizeEMT && height <= maxSizeEMT && sps.getUseIntraMTS() )
  {
#if JVET_AH0103_LOW_DELAY_LFNST_NSPT
    mtsUsageFlag = ( spsIntraLfnstEnabled && cu.mtsFlag == 1 ) ? 2 : 1;
#else
    mtsUsageFlag = ( sps.getUseLFNST() && cu.mtsFlag == 1 ) ? 2 : 1;
#endif
  }

  if( width * height < 64 && !m_pcEncCfg->getUseFastLFNST() )
  {
    mtsUsageFlag = 0;
  }

#if JVET_W0103_INTRA_MTS
  if (!cu.mtsFlag && !cu.lfnstIdx)
  {
    m_globalBestCostStore = MAX_DOUBLE;
    m_globalBestCostValid = false;
    if (bestCS->getCU(partitioner.chType) != NULL && bestCS->getCU(partitioner.chType)->predMode != MODE_INTRA && bestCostSoFar != MAX_DOUBLE)
    {
      m_globalBestCostStore = bestCostSoFar;
      m_globalBestCostValid = true;
    }
#if JVET_Y0142_ADAPT_INTRA_MTS
    m_modesForMTS.clear();
    m_modesCoeffAbsSumDCT2.clear();
#endif
#if JVET_AE0169_BIPREDICTIVE_IBC
    m_bestIntraSADHADCost = MAX_DOUBLE;
#endif
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
    m_bestIntraSADCost = MAX_DOUBLE;
#endif
  }
#endif

  const bool colorTransformIsEnabled = sps.getUseColorTrans() && !CS::isDualITree(cs);
  const bool isFirstColorSpace       = colorTransformIsEnabled && ((m_pcEncCfg->getRGBFormatFlag() && cu.colorTransform) || (!m_pcEncCfg->getRGBFormatFlag() && !cu.colorTransform));
  const bool isSecondColorSpace      = colorTransformIsEnabled && ((m_pcEncCfg->getRGBFormatFlag() && !cu.colorTransform) || (!m_pcEncCfg->getRGBFormatFlag() && cu.colorTransform));

  double bestCurrentCost = bestCostSoFar;
  bool ispCanBeUsed   = sps.getUseISP() && cu.mtsFlag == 0 && cu.lfnstIdx == 0 && CU::canUseISP(width, height, cu.cs->sps->getMaxTbSize());
  bool saveDataForISP = ispCanBeUsed && (!colorTransformIsEnabled || isFirstColorSpace);
  bool testISP        = ispCanBeUsed && (!colorTransformIsEnabled || !cu.colorTransform);

#if JVET_AB0155_SGPM
  const bool sgpmAllowed = sps.getUseSgpm() && isLuma(partitioner.chType);
  bool testSgpm = sgpmAllowed && cu.lwidth() >= GEO_MIN_CU_SIZE_EX && cu.lheight() >= GEO_MIN_CU_SIZE_EX
                  && cu.lwidth() <= GEO_MAX_CU_SIZE_EX && cu.lheight() <= GEO_MAX_CU_SIZE_EX
                  && cu.lwidth() < 8 * cu.lheight() && cu.lheight() < 8 * cu.lwidth() && cu.lx() && cu.ly()
                  && cu.lwidth() * cu.lheight() >= SGPM_MIN_PIX;
#endif
#if JVET_AG0058_EIP
  bool testEip = isLuma(partitioner.chType) && sps.getUseEip() && (getAllowedEip(cu, COMPONENT_Y) || getAllowedEipMerge(cu, COMPONENT_Y));
  bool eipSaveFlag = (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
#if JVET_AJ0082_MM_EIP
  double eipCost = MAX_DOUBLE;
  bool setSkipEipControl = (m_pcEncCfg->getIntraPeriod() == 1) && !cu.lfnstIdx && !cu.mtsFlag;
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
  CodedCUInfo *relatedCU = ((EncModeCtrlMTnoRQT *) m_modeCtrl)->getBlkInfoPtr(partitioner.currArea());
  testEip &= !relatedCU->skipEip;
#else
  CodedCUInfo &relatedCU = ((EncModeCtrlMTnoRQT *) m_modeCtrl)->getBlkInfo(partitioner.currArea());
  testEip &= !relatedCU.skipEip;
#endif
  double eipBestSatdCost = MAX_DOUBLE;
  bool isEipModeTested = false;
#endif
#endif