Commit 320db0c7 authored by Xiang Li's avatar Xiang Li

Merge branch 'JVET_N0193_LFNSTEncFix' into 'master'

LFNST encoding bug-fix

See merge request jvet/VVCSoftware_VTM!532
parents b9d568d8 a953dcc9
Pipeline #1502 canceled with stage
......@@ -1343,11 +1343,17 @@ void IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner,
if (testMip)
{
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTemp;
#if JVET_N0193_LFNST
static_vector<int, FAST_UDI_MAX_RDMODE_NUM> rdModeIdxList;
#endif
for( int i = 0; i < uiRdModeList.size(); i++)
{
if( !uiRdModeList[i].mipFlg && uiRdModeList[i].ispMod==NOT_INTRA_SUBPARTITIONS )
{
uiRdModeListTemp.push_back( uiRdModeList[i] );
#if JVET_N0193_LFNST
rdModeIdxList.push_back( i );
#endif
}
}
for( int i = 0; i < uiRdModeList.size(); i++)
......@@ -1355,6 +1361,9 @@ void IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner,
if( uiRdModeList[i].mipFlg || uiRdModeList[i].ispMod!=NOT_INTRA_SUBPARTITIONS )
{
uiRdModeListTemp.push_back( uiRdModeList[i] );
#if JVET_N0193_LFNST
rdModeIdxList.push_back( i );
#endif
}
}
for( int i = 0; i < uiRdModeList.size(); i++)
......@@ -1559,11 +1568,19 @@ void IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner,
#if JVET_N0413_RDPCM
if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode && mode >= 0 )
{
#if JVET_N0217_MATRIX_INTRAPRED
m_modeCostStore[ lfnstIdx ][ rdModeIdxList[ mode ] ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
#else
m_modeCostStore[ lfnstIdx ][ mode ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
#endif
#else
if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode )
{
#if JVET_N0217_MATRIX_INTRAPRED
m_modeCostStore[ lfnstIdx ][ rdModeIdxList[ uiMode ] ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
#else
m_modeCostStore[ lfnstIdx ][ uiMode ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
#endif
#endif
}
#endif
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment