Commit d3941ac2 authored by Moonmo Koo's avatar Moonmo Koo Committed by Frank Bossen
Browse files

JVET-N0193: Low Frequency Non-Separable Transform (LFNST)

- Previous name: Reduced Secondary Transform (RST)
- The parameter, LFNST, should be added in cfg file for enabing LFNST, i.e. LFNST = 1
parent 3f2b49e9
......@@ -98,6 +98,7 @@ MTS : 1
MTSIntraMaxCand : 3
MTSInterMaxCand : 4
SBT : 1
LFNST : 1
ISP : 1
Affine : 1
SubPuMvp : 1
......@@ -118,6 +119,7 @@ ISPFast : 1
FastMrg : 1
AMaxBT : 1
FastMIP : 1
FastLFNST : 1
# Encoder optimization tools
AffineAmvrEncOpt : 0
......
......@@ -128,6 +128,7 @@ MTS : 1
MTSIntraMaxCand : 3
MTSInterMaxCand : 4
SBT : 1
LFNST : 1
ISP : 1
MMVD : 1
Affine : 1
......@@ -156,6 +157,7 @@ ISPFast : 1
FastMrg : 1
AMaxBT : 1
FastMIP : 0
FastLFNST : 0
# Encoder optimization tools
AffineAmvrEncOpt : 1
......
......@@ -2040,6 +2040,18 @@ Enables or disables symmetric MVD mode.
Enables or disables RDPCM coding mode.
\\
\Option{LFNST} &
%\ShortOption{\None} &
\Default{false} &
Enables or disables the use of low frequency non-separable transform (LFNST).
\\
\Option{FastLFNST} &
%\ShortOption{\None} &
\Default{false} &
Enables or disables the fast encoding of low frequency non-separable transform (LFNST).
\\
\end{OptionTableNoShorthand}
%%
......
......@@ -248,6 +248,10 @@ void EncApp::xInitLibCfg()
m_cEncLib.setMinQTSizes ( m_uiMinQT );
m_cEncLib.setMaxBTDepth ( m_uiMaxBTDepth, m_uiMaxBTDepthI, m_uiMaxBTDepthIChroma );
m_cEncLib.setDualITree ( m_dualTree );
#if JVET_N0193_LFNST
m_cEncLib.setLFNST ( m_LFNST );
m_cEncLib.setUseFastLFNST ( m_useFastLFNST );
#endif
m_cEncLib.setSubPuMvpMode ( m_SubPuMvpMode );
m_cEncLib.setAffine ( m_Affine );
m_cEncLib.setAffineType ( m_AffineType );
......
......@@ -834,6 +834,10 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] )
("MaxBTDepthISliceL", m_uiMaxBTDepthI, 3u, "MaxBTDepthISliceL")
("MaxBTDepthISliceC", m_uiMaxBTDepthIChroma, 3u, "MaxBTDepthISliceC")
("DualITree", m_dualTree, false, "Use separate QTBT trees for intra slice luma and chroma channel types")
#if JVET_N0193_LFNST
( "LFNST", m_LFNST, false, "Enable LFNST (0:off, 1:on) [default: off]" )
( "FastLFNST", m_useFastLFNST, false, "Fast methods for LFNST" )
#endif
("SubPuMvp", m_SubPuMvpMode, 0, "Enable Sub-PU temporal motion vector prediction (0:off, 1:ATMVP, 2:STMVP, 3:ATMVP+STMVP) [default: off]")
#if JVET_N0127_MMVD_SPS_FLAG
("MMVD", m_MMVD, true, "Enable Merge mode with Motion Vector Difference (0:off, 1:on) [default: 1]")
......@@ -3159,6 +3163,9 @@ void EncAppCfg::xPrintParameter()
if( m_profile == Profile::NEXT )
{
msg( VERBOSE, "\nNEXT TOOL CFG: " );
#if JVET_N0193_LFNST
msg( VERBOSE, "LFNST:%d ", m_LFNST );
#endif
#if JVET_N0127_MMVD_SPS_FLAG
msg( VERBOSE, "MMVD:%d ", m_MMVD);
#endif
......@@ -3238,6 +3245,9 @@ void EncAppCfg::xPrintParameter()
if( m_ISP ) msg( VERBOSE, "ISPFast:%d ", m_useFastISP );
#else
msg( VERBOSE, "ISPFast:%d ", m_useFastISP );
#endif
#if JVET_N0193_LFNST
if( m_LFNST ) msg( VERBOSE, "FastLFNST:%d ", m_useFastLFNST );
#endif
msg( VERBOSE, "AMaxBT:%d ", m_useAMaxBT );
msg( VERBOSE, "E0023FastEnc:%d ", m_e0023FastEnc );
......
......@@ -238,6 +238,10 @@ protected:
unsigned m_uiMaxBTDepthI;
unsigned m_uiMaxBTDepthIChroma;
bool m_dualTree;
#if JVET_N0193_LFNST
bool m_LFNST;
bool m_useFastLFNST;
#endif
int m_SubPuMvpMode;
bool m_Affine;
bool m_AffineType;
......
......@@ -91,6 +91,9 @@ enum CodingStatisticsType
STATS__CABAC_BITS__SAO,
STATS__CABAC_BITS__ALF,
STATS__CABAC_TRM_BITS,
#if JVET_N0193_LFNST
STATS__CABAC_BITS__LFNST,
#endif
STATS__CABAC_FIXED_BITS,
STATS__CABAC_PCM_ALIGN_BITS,
STATS__CABAC_PCM_CODE_BITS,
......@@ -109,6 +112,9 @@ enum CodingStatisticsType
STATS__CABAC_BITS__TRIANGLE_FLAG,
STATS__CABAC_BITS__TRIANGLE_INDEX,
STATS__CABAC_BITS__MULTI_REF_LINE,
#if JVET_N0193_LFNST
STATS__TOOL_LFNST,
#endif
STATS__CABAC_BITS__SYMMVD_FLAG,
STATS__TOOL_TOTAL_FRAME,// This is a special case and is not included in the report.
STATS__TOOL_AFF,
......@@ -173,6 +179,9 @@ static inline const char* getName(CodingStatisticsType name)
"CABAC_BITS__SIGN_BIT",
"CABAC_BITS__ESCAPE_BITS",
"CABAC_BITS__SAO",
#if JVET_N0193_LFNST
"CABAC_BITS__LFNST",
#endif
"CABAC_BITS__ALF",
"CABAC_TRM_BITS",
"CABAC_FIXED_BITS",
......@@ -197,6 +206,9 @@ static inline const char* getName(CodingStatisticsType name)
"TOOL_FRAME",
"TOOL_AFFINE",
"TOOL_EMT",
#if JVET_N0193_LFNST
"TOOL_LFNST",
#endif
"TOOL_TOTAL"
};
CHECK( STATS__NUM_STATS != sizeof( statNames ) / sizeof( char* ) || name >= STATS__NUM_STATS, "stats out of range" );
......
......@@ -217,6 +217,10 @@ static const int NUM_LUMA_MODE = 67; ///< Plan
static const int NUM_LMC_MODE = 1 + 2; ///< LMC + MDLM_T + MDLM_L
static const int NUM_INTRA_MODE = (NUM_LUMA_MODE + NUM_LMC_MODE);
#if JVET_N0193_LFNST
static const int NUM_EXT_LUMA_MODE = 28;
#endif
static const int NUM_DIR = (((NUM_LUMA_MODE - 3) >> 2) + 1);
static const int PLANAR_IDX = 0; ///< index for intra PLANAR mode
static const int DC_IDX = 1; ///< index for intra DC mode
......@@ -253,6 +257,15 @@ static const int FAST_UDI_MAX_RDMODE_NUM = (NUM_LUMA_MODE + MAX_NUM_MIP_MODE); /
static const int FAST_UDI_MAX_RDMODE_NUM = NUM_LUMA_MODE; ///< maximum number of RD comparison in fast-UDI estimation loop
#endif
#if JVET_N0193_LFNST
static const int MAX_LFNST_COEF_NUM = 16;
static const int LFNST_SIG_NZ_LUMA = 1;
static const int LFNST_SIG_NZ_CHROMA = 1;
static const int NUM_LFNST_NUM_PER_SET = 3;
#endif
static const int MDCS_ANGLE_LIMIT = 9; ///< 0 = Horizontal/vertical only, 1 = Horizontal/vertical +/- 1, 2 = Horizontal/vertical +/- 2 etc...
static const int MDCS_MAXIMUM_WIDTH = 8; ///< (measured in pixels) TUs with width greater than this can only use diagonal scan
......
......@@ -693,6 +693,16 @@ const CtxSet ContextSetCfg::TransquantBypassFlag = ContextSetCfg::addCtxSet
{ DWS, }
});
#if JVET_N0193_LFNST
const CtxSet ContextSetCfg::LFNSTIdx = ContextSetCfg::addCtxSet
( {
{ CNU, CNU, CNU, CNU, CNU, CNU, CNU, CNU, },
{ CNU, CNU, CNU, CNU, CNU, CNU, CNU, CNU, },
{ CNU, CNU, CNU, CNU, CNU, CNU, CNU, CNU, },
{ DWS, DWS, DWS, DWS, DWS, DWS, DWS, DWS, },
} );
#endif
const CtxSet ContextSetCfg::RdpcmFlag = ContextSetCfg::addCtxSet
({
{ 139, 139,},
......
......@@ -250,6 +250,9 @@ public:
static const CtxSet SaoTypeIdx;
static const CtxSet MTSIndex;
static const CtxSet TransquantBypassFlag;
#if JVET_N0193_LFNST
static const CtxSet LFNSTIdx;
#endif
static const CtxSet RdpcmFlag;
static const CtxSet RdpcmDir;
static const CtxSet SbtFlag;
......
......@@ -1003,75 +1003,119 @@ namespace DQIntern
m_goRiceZero = 0;
}
#if JVET_N0193_LFNST
void checkRdCosts( const ScanPosType spt, const PQData& pqDataA, const PQData& pqDataB, Decision& decisionA, Decision& decisionB, bool zeroOut ) const
#else
void checkRdCosts( const ScanPosType spt, const PQData& pqDataA, const PQData& pqDataB, Decision& decisionA, Decision& decisionB) const
#endif
{
const int32_t* goRiceTab = g_goRiceBits[m_goRicePar];
int64_t rdCostA = m_rdCost + pqDataA.deltaDist;
int64_t rdCostB = m_rdCost + pqDataB.deltaDist;
int64_t rdCostZ = m_rdCost;
if( m_remRegBins >= 4 )
#if JVET_N0193_LFNST
if( zeroOut )
{
if( pqDataA.absLevel < 4 )
rdCostA += m_coeffFracBits.bits[pqDataA.absLevel];
else
rdCostZ = m_rdCost;
if( m_remRegBins >= 4 )
{
const unsigned value = (pqDataA.absLevel - 4) >> 1;
rdCostA += m_coeffFracBits.bits[pqDataA.absLevel - (value << 1)] + goRiceTab[value<RICEMAX ? value : RICEMAX-1];
if( spt == SCAN_ISCSBB )
{
rdCostZ += m_sigFracBits.intBits[ 0 ];
}
else if( spt == SCAN_SOCSBB )
{
rdCostZ += m_sbbFracBits.intBits[ 1 ] + m_sigFracBits.intBits[ 0 ];
}
else if( m_numSigSbb )
{
rdCostZ += m_sigFracBits.intBits[ 0 ];
}
else
{
rdCostZ = decisionA.rdCost;
}
}
if( pqDataB.absLevel < 4 )
rdCostB += m_coeffFracBits.bits[pqDataB.absLevel];
else
{
const unsigned value = (pqDataB.absLevel - 4) >> 1;
rdCostB += m_coeffFracBits.bits[pqDataB.absLevel - (value << 1)] + goRiceTab[value<RICEMAX ? value : RICEMAX-1];
rdCostZ += goRiceTab[ m_goRiceZero ];
}
if( spt == SCAN_ISCSBB )
if( rdCostZ < decisionA.rdCost )
{
rdCostA += m_sigFracBits.intBits[1];
rdCostB += m_sigFracBits.intBits[1];
rdCostZ += m_sigFracBits.intBits[0];
decisionA.rdCost = rdCostZ;
decisionA.absLevel = 0;
decisionA.prevId = m_stateId;
}
else if( spt == SCAN_SOCSBB )
}
else
{
#endif
if( m_remRegBins >= 4 )
{
rdCostA += m_sbbFracBits.intBits[1] + m_sigFracBits.intBits[1];
rdCostB += m_sbbFracBits.intBits[1] + m_sigFracBits.intBits[1];
rdCostZ += m_sbbFracBits.intBits[1] + m_sigFracBits.intBits[0];
if( pqDataA.absLevel < 4 )
rdCostA += m_coeffFracBits.bits[ pqDataA.absLevel ];
else
{
const unsigned value = ( pqDataA.absLevel - 4 ) >> 1;
rdCostA += m_coeffFracBits.bits[ pqDataA.absLevel - ( value << 1 ) ] + goRiceTab[ value < RICEMAX ? value : RICEMAX - 1 ];
}
if( pqDataB.absLevel < 4 )
rdCostB += m_coeffFracBits.bits[ pqDataB.absLevel ];
else
{
const unsigned value = ( pqDataB.absLevel - 4 ) >> 1;
rdCostB += m_coeffFracBits.bits[ pqDataB.absLevel - ( value << 1 ) ] + goRiceTab[ value < RICEMAX ? value : RICEMAX - 1 ];
}
if( spt == SCAN_ISCSBB )
{
rdCostA += m_sigFracBits.intBits[ 1 ];
rdCostB += m_sigFracBits.intBits[ 1 ];
rdCostZ += m_sigFracBits.intBits[ 0 ];
}
else if( spt == SCAN_SOCSBB )
{
rdCostA += m_sbbFracBits.intBits[ 1 ] + m_sigFracBits.intBits[ 1 ];
rdCostB += m_sbbFracBits.intBits[ 1 ] + m_sigFracBits.intBits[ 1 ];
rdCostZ += m_sbbFracBits.intBits[ 1 ] + m_sigFracBits.intBits[ 0 ];
}
else if( m_numSigSbb )
{
rdCostA += m_sigFracBits.intBits[ 1 ];
rdCostB += m_sigFracBits.intBits[ 1 ];
rdCostZ += m_sigFracBits.intBits[ 0 ];
}
else
{
rdCostZ = decisionA.rdCost;
}
}
else if( m_numSigSbb )
else
{
rdCostA += m_sigFracBits.intBits[1];
rdCostB += m_sigFracBits.intBits[1];
rdCostZ += m_sigFracBits.intBits[0];
rdCostA += ( 1 << SCALE_BITS ) + goRiceTab[ pqDataA.absLevel <= m_goRiceZero ? pqDataA.absLevel - 1 : ( pqDataA.absLevel < RICEMAX ? pqDataA.absLevel : RICEMAX - 1 ) ];
rdCostB += ( 1 << SCALE_BITS ) + goRiceTab[ pqDataB.absLevel <= m_goRiceZero ? pqDataB.absLevel - 1 : ( pqDataB.absLevel < RICEMAX ? pqDataB.absLevel : RICEMAX - 1 ) ];
rdCostZ += goRiceTab[ m_goRiceZero ];
}
else
if( rdCostA < decisionA.rdCost )
{
rdCostZ = decisionA.rdCost;
decisionA.rdCost = rdCostA;
decisionA.absLevel = pqDataA.absLevel;
decisionA.prevId = m_stateId;
}
if( rdCostZ < decisionA.rdCost )
{
decisionA.rdCost = rdCostZ;
decisionA.absLevel = 0;
decisionA.prevId = m_stateId;
}
if( rdCostB < decisionB.rdCost )
{
decisionB.rdCost = rdCostB;
decisionB.absLevel = pqDataB.absLevel;
decisionB.prevId = m_stateId;
}
#if JVET_N0193_LFNST
}
else
{
rdCostA += (1 << SCALE_BITS) + goRiceTab[pqDataA.absLevel <= m_goRiceZero ? pqDataA.absLevel - 1 : (pqDataA.absLevel<RICEMAX ? pqDataA.absLevel : RICEMAX-1)];
rdCostB += (1 << SCALE_BITS) + goRiceTab[pqDataB.absLevel <= m_goRiceZero ? pqDataB.absLevel - 1 : (pqDataB.absLevel<RICEMAX ? pqDataB.absLevel : RICEMAX-1)];
rdCostZ += goRiceTab[m_goRiceZero];
}
if( rdCostA < decisionA.rdCost )
{
decisionA.rdCost = rdCostA;
decisionA.absLevel = pqDataA.absLevel;
decisionA.prevId = m_stateId;
}
if( rdCostZ < decisionA.rdCost )
{
decisionA.rdCost = rdCostZ;
decisionA.absLevel = 0;
decisionA.prevId = m_stateId;
}
if( rdCostB < decisionB.rdCost )
{
decisionB.rdCost = rdCostB;
decisionB.absLevel = pqDataB.absLevel;
decisionB.prevId = m_stateId;
}
#endif
}
inline void checkRdCostStart(int32_t lastOffset, const PQData &pqData, Decision &decision) const
......@@ -1479,6 +1523,7 @@ namespace DQIntern
{
::memcpy( decisions, startDec, 8*sizeof(Decision) );
#if !JVET_N0193_LFNST
if( zeroOut )
{
if( spt==SCAN_EOCSBB )
......@@ -1490,22 +1535,51 @@ namespace DQIntern
}
return;
}
#endif
PQData pqData[4];
m_quant.preQuantCoeff( absCoeff, pqData );
#if JVET_N0193_LFNST
m_prevStates[0].checkRdCosts( spt, pqData[0], pqData[2], decisions[0], decisions[2], zeroOut );
m_prevStates[1].checkRdCosts( spt, pqData[0], pqData[2], decisions[2], decisions[0], zeroOut );
m_prevStates[2].checkRdCosts( spt, pqData[3], pqData[1], decisions[1], decisions[3], zeroOut );
m_prevStates[3].checkRdCosts( spt, pqData[3], pqData[1], decisions[3], decisions[1], zeroOut );
#else
m_prevStates[0].checkRdCosts( spt, pqData[0], pqData[2], decisions[0], decisions[2]);
m_prevStates[1].checkRdCosts( spt, pqData[0], pqData[2], decisions[2], decisions[0]);
m_prevStates[2].checkRdCosts( spt, pqData[3], pqData[1], decisions[1], decisions[3]);
m_prevStates[3].checkRdCosts( spt, pqData[3], pqData[1], decisions[3], decisions[1]);
#endif
if( spt==SCAN_EOCSBB )
{
m_skipStates[0].checkRdCostSkipSbb( decisions[0] );
m_skipStates[1].checkRdCostSkipSbb( decisions[1] );
m_skipStates[2].checkRdCostSkipSbb( decisions[2] );
m_skipStates[3].checkRdCostSkipSbb( decisions[3] );
#if JVET_N0193_LFNST
if( zeroOut )
{
m_skipStates[0].checkRdCostSkipSbbZeroOut( decisions[0] );
m_skipStates[1].checkRdCostSkipSbbZeroOut( decisions[1] );
m_skipStates[2].checkRdCostSkipSbbZeroOut( decisions[2] );
m_skipStates[3].checkRdCostSkipSbbZeroOut( decisions[3] );
}
else
{
#endif
m_skipStates[0].checkRdCostSkipSbb( decisions[0] );
m_skipStates[1].checkRdCostSkipSbb( decisions[1] );
m_skipStates[2].checkRdCostSkipSbb( decisions[2] );
m_skipStates[3].checkRdCostSkipSbb( decisions[3] );
#if JVET_N0193_LFNST
}
#endif
}
#if JVET_N0193_LFNST
if( !zeroOut )
{
#endif
m_startState.checkRdCostStart( lastOffset, pqData[0], decisions[0] );
m_startState.checkRdCostStart( lastOffset, pqData[2], decisions[2] );
#if JVET_N0193_LFNST
}
#endif
}
void DepQuant::xDecideAndUpdate( const TCoeff absCoeff, const ScanInfo& scanInfo, bool zeroOut )
......@@ -1527,7 +1601,11 @@ namespace DQIntern
m_currStates[3].updateStateEOS( scanInfo, m_prevStates, m_skipStates, decisions[3] );
::memcpy( decisions+4, decisions, 4*sizeof(Decision) );
}
#if !JVET_N0193_LFNST
else if( !zeroOut )
#else
else
#endif
{
switch( scanInfo.nextNbInfoSbb.num )
{
......@@ -1590,8 +1668,21 @@ namespace DQIntern
::memset( tu.getCoeffs( compID ).buf, 0x00, numCoeff*sizeof(TCoeff) );
absSum = 0;
#if JVET_N0193_LFNST
const CompArea& area = tu.blocks[ compID ];
const uint32_t width = area.width;
const uint32_t height = area.height;
const uint32_t lfnstIdx = tu.cu->lfnstIdx;
#endif
//===== find first test position =====
int firstTestPos = numCoeff - 1;
#if JVET_N0193_LFNST
if( lfnstIdx > 0 && tu.mtsIdx != 1 && ( ( width == 4 && height == 4 ) || ( width == 8 && height == 8 ) ) )
{
firstTestPos = 7;
}
#endif
const TCoeff thres = m_quant.getLastThreshold();
for( ; firstTestPos >= 0; firstTestPos-- )
{
......@@ -1627,7 +1718,13 @@ namespace DQIntern
for( int scanIdx = firstTestPos; scanIdx >= 0; scanIdx-- )
{
const ScanInfo& scanInfo = tuPars.m_scanInfo[ scanIdx ];
#if JVET_N0193_LFNST
bool lfnstZeroOut = lfnstIdx > 0 && tu.mtsIdx != 1 && width >= 4 && height >= 4 &&
( ( ( ( width >= 8 && height >= 8 ) && scanIdx >= 16 ) || ( ( ( width == 4 && height == 4 ) || ( width == 8 && height == 8 ) ) && scanIdx >= 8 ) ) && scanIdx < 48 );
xDecideAndUpdate( abs( tCoeff[ scanInfo.rasterPos ] ), scanInfo, ( zeroOut && ( scanInfo.posX >= effWidth || scanInfo.posY >= effHeight ) ) || lfnstZeroOut );
#else
xDecideAndUpdate( abs( tCoeff[ scanInfo.rasterPos ] ), scanInfo, zeroOut && ( scanInfo.posX >= effWidth || scanInfo.posY >= effHeight ) );
#endif
}
//===== find best path =====
......
......@@ -165,7 +165,6 @@ public:
/// set parameters from CU data for accessing intra data
void initIntraPatternChType (const CodingUnit &cu, const CompArea &area, const bool forceRefFilterFlag = false); // use forceRefFilterFlag to get both filtered and unfiltered buffers
#if JVET_N0217_MATRIX_INTRAPRED
// Matrix-based intra prediction
void initIntraMip (const PredictionUnit &pu);
......@@ -173,6 +172,9 @@ public:
#endif
static bool useFilteredIntraRefSamples( const ComponentID &compID, const PredictionUnit &pu, bool modeSpecific, const UnitArea &tuArea );
#if HM_MDIS_AS_IN_JEM && JVET_N0193_LFNST
static bool getPlanarMDISCondition( const UnitArea &tuArea ) { return abs( PLANAR_IDX - HOR_IDX ) > m_aucIntraFilter[ CHANNEL_TYPE_LUMA ][ ( ( g_aucLog2[ tuArea.Y().width ] + g_aucLog2[ tuArea.Y().height ] ) >> 1 ) ]; }
#endif
static bool useDPCMForFirstPassIntraEstimation(const PredictionUnit &pu, const uint32_t &uiDirMode);
void geneWeightedPred (const ComponentID compId, PelBuf &pred, const PredictionUnit &pu, Pel *srcBuf);
......
......@@ -759,15 +759,36 @@ void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID,
DTRACE( g_trace_ctx, D_RDOQ, "%d: %3d, %3d, %dx%d, comp=%d\n", DTRACE_GET_COUNTER( g_trace_ctx, D_RDOQ ), rect.x, rect.y, rect.width, rect.height, compID );
#endif
#if JVET_N0193_LFNST
const uint32_t lfnstIdx = tu.cu->lfnstIdx;
#endif
for (int subSetId = iCGNum - 1; subSetId >= 0; subSetId--)
{
cctx.initSubblock( subSetId );
#if JVET_N0193_LFNST
uint32_t maxNonZeroPosInCG = iCGSizeM1;
if( lfnstIdx > 0 && ( ( uiWidth == 4 && uiHeight == 4 ) || ( uiWidth == 8 && uiHeight == 8 && cctx.cgPosX() == 0 && cctx.cgPosY() == 0 ) ) )
{
maxNonZeroPosInCG = 7;
}
#endif
memset( &rdStats, 0, sizeof (coeffGroupRDStats));
#if JVET_N0193_LFNST
for( int iScanPosinCG = iCGSizeM1; iScanPosinCG > maxNonZeroPosInCG; iScanPosinCG-- )
{
iScanPos = cctx.minSubPos() + iScanPosinCG;
uint32_t blkPos = cctx.blockPos( iScanPos );
piDstCoeff[ blkPos ] = 0;
}
for( int iScanPosinCG = maxNonZeroPosInCG; iScanPosinCG >= 0; iScanPosinCG-- )
#else
for (int iScanPosinCG = iCGSizeM1; iScanPosinCG >= 0; iScanPosinCG--)
#endif
{
iScanPos = cctx.minSubPos() + iScanPosinCG;
//===== quantization =====
......@@ -970,7 +991,11 @@ void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID,
pdCostCoeffGroupSig[ cctx.subSetId() ] = xGetRateSigCoeffGroup(fracBitsSigGroup,0);
}
// reset coeffs to 0 in this block
#if JVET_N0193_LFNST
for( int iScanPosinCG = maxNonZeroPosInCG; iScanPosinCG >= 0; iScanPosinCG-- )
#else
for (int iScanPosinCG = iCGSizeM1; iScanPosinCG >= 0; iScanPosinCG--)
#endif
{
iScanPos = cctx.minSubPos() + iScanPosinCG;
uint32_t uiBlkPos = cctx.blockPos( iScanPos );
......@@ -1088,7 +1113,16 @@ void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID,
d64BaseCost -= pdCostCoeffGroupSig [ iCGScanPos ];
if (cctx.isSigGroup( iCGScanPos ) )
{
#if JVET_N0193_LFNST
uint32_t maxNonZeroPosInCG = iCGSizeM1;
if( lfnstIdx > 0 && ( ( uiWidth == 4 && uiHeight == 4 ) || ( uiWidth == 8 && uiHeight == 8 && cctx.cgPosX() == 0 && cctx.cgPosY() == 0 ) ) )
{
maxNonZeroPosInCG = 7;
}
for( int iScanPosinCG = maxNonZeroPosInCG; iScanPosinCG >= 0; iScanPosinCG-- )
#else
for (int iScanPosinCG = iCGSizeM1; iScanPosinCG >= 0; iScanPosinCG--)
#endif
{
iScanPos = iCGScanPos * (iCGSizeM1 + 1) + iScanPosinCG;
......
......@@ -467,6 +467,32 @@ void initROM()
}
#endif
#if JVET_N0193_LFNST
// initialize CoefTopLeftDiagScan8x8 for LFNST
for( uint32_t blockWidthIdx = 0; blockWidthIdx < sizeInfo.numAllWidths(); blockWidthIdx++ )
{
const uint32_t blockWidth = sizeInfo.sizeFrom( blockWidthIdx );
const static uint8_t g_auiXYDiagScan8x8[ 64 ][ 2 ] =
{
{ 0, 0 }, { 0, 1 }, { 1, 0 }, { 0, 2 }, { 1, 1 }, { 2, 0 }, { 0, 3 }, { 1, 2 },
{ 2, 1 }, { 3, 0 }, { 1, 3 }, { 2, 2 }, { 3, 1 }, { 2, 3 }, { 3, 2 }, { 3, 3 },
{ 0, 4 }, { 0, 5 }, { 1, 4 }, { 0, 6 }, { 1, 5 }, { 2, 4 }, { 0, 7 }, { 1, 6 },
{ 2, 5 }, { 3, 4 }, { 1, 7 }, { 2, 6 }, { 3, 5 }, { 2, 7 }, { 3, 6 }, { 3, 7 },
{ 4, 0 }, { 4, 1 }, { 5, 0 }, { 4, 2 }, { 5, 1 }, { 6, 0 }, { 4, 3 }, { 5, 2 },
{ 6, 1 }, { 7, 0 }, { 5, 3 }, { 6, 2 }, { 7, 1 }, { 6, 3 }, { 7, 2 }, { 7, 3 },