From 6f6e20ca55e58d072f6973554f2caed12093258e Mon Sep 17 00:00:00 2001
From: Yin Zhao <yin.zhao@huawei.com>
Date: Wed, 13 Feb 2019 19:28:49 +0100
Subject: [PATCH] Fix ticket #185 on compilation error with JVET_M0140_SBT when
JVET_M0464_UNI_MTS is disabled
---
source/Lib/CommonLib/TrQuant.cpp | 10 +++++
source/Lib/DecoderLib/CABACReader.cpp | 16 ++++++++
source/Lib/EncoderLib/CABACWriter.cpp | 15 +++++++
source/Lib/EncoderLib/EncCu.cpp | 56 +++++++++++++++++++++++++++
source/Lib/EncoderLib/InterSearch.cpp | 12 ++++++
5 files changed, 109 insertions(+)
diff --git a/source/Lib/CommonLib/TrQuant.cpp b/source/Lib/CommonLib/TrQuant.cpp
index 44ed81e06..1972b1577 100644
--- a/source/Lib/CommonLib/TrQuant.cpp
+++ b/source/Lib/CommonLib/TrQuant.cpp
@@ -320,8 +320,13 @@ void TrQuant::getTrTypes ( TransformUnit tu, const ComponentID compID, int &trTy
if( sbtIdx == SBT_VER_HALF || sbtIdx == SBT_VER_QUAD )
{
+#if JVET_M0464_UNI_MTS
assert( tu.lwidth() <= MTS_INTER_MAX_CU_SIZE );
if( tu.lheight() > MTS_INTER_MAX_CU_SIZE )
+#else
+ assert( tu.lwidth() <= EMT_INTER_MAX_CU_WITH_QTBT );
+ if( tu.lheight() > EMT_INTER_MAX_CU_WITH_QTBT )
+#endif
{
trTypeHor = trTypeVer = DCT2;
}
@@ -333,8 +338,13 @@ void TrQuant::getTrTypes ( TransformUnit tu, const ComponentID compID, int &trTy
}
else
{
+#if JVET_M0464_UNI_MTS
assert( tu.lheight() <= MTS_INTER_MAX_CU_SIZE );
if( tu.lwidth() > MTS_INTER_MAX_CU_SIZE )
+#else
+ assert( tu.lheight() <= EMT_INTER_MAX_CU_WITH_QTBT );
+ if( tu.lwidth() > EMT_INTER_MAX_CU_WITH_QTBT )
+#endif
{
trTypeHor = trTypeVer = DCT2;
}
diff --git a/source/Lib/DecoderLib/CABACReader.cpp b/source/Lib/DecoderLib/CABACReader.cpp
index 4fb88ca5c..db7838562 100644
--- a/source/Lib/DecoderLib/CABACReader.cpp
+++ b/source/Lib/DecoderLib/CABACReader.cpp
@@ -2812,6 +2812,9 @@ void CABACReader::residual_coding( TransformUnit& tu, ComponentID compID )
#if JVET_M0102_INTRA_SUBPARTITIONS
useEmt = useEmt && !cu.ispMode;
#endif
+#if JVET_M0140_SBT
+ useEmt = useEmt && !cu.sbtInfo;
+#endif
#endif
for( int subSetId = ( cctx.scanPosLast() >> cctx.log2CGSize() ); subSetId >= 0; subSetId--)
@@ -2922,6 +2925,13 @@ void CABACReader::transform_skip_flag( TransformUnit& tu, ComponentID compID )
tu.transformSkip[compID] = false;
return;
}
+#if JVET_M0140_SBT
+ if( tu.cu->sbtInfo )
+ {
+ tu.transformSkip[compID] = false;
+ return;
+ }
+#endif
RExt__DECODER_DEBUG_BIT_STATISTICS_CREATE_SET2( STATS__CABAC_BITS__TRANSFORM_SKIP_FLAGS, compID );
bool tskip = m_BinDecoder.decodeBin( Ctx::TransformSkipFlag( toChannelType( compID ) ) );
@@ -2966,6 +2976,12 @@ void CABACReader::emt_cu_flag( CodingUnit& cu )
{
return;
}
+#endif
+#if JVET_M0140_SBT
+ if( cu.sbtInfo )
+ {
+ return;
+ }
#endif
const CodingStructure &cs = *cu.cs;
diff --git a/source/Lib/EncoderLib/CABACWriter.cpp b/source/Lib/EncoderLib/CABACWriter.cpp
index 372cf7ae5..a9b4f7aef 100644
--- a/source/Lib/EncoderLib/CABACWriter.cpp
+++ b/source/Lib/EncoderLib/CABACWriter.cpp
@@ -2664,6 +2664,9 @@ void CABACWriter::residual_coding( const TransformUnit& tu, ComponentID compID )
#if JVET_M0102_INTRA_SUBPARTITIONS
useEmt = useEmt && !cu.ispMode;
#endif
+#if JVET_M0140_SBT
+ useEmt = useEmt && !cu.sbtInfo;
+#endif
#endif
for( int subSetId = ( cctx.scanPosLast() >> cctx.log2CGSize() ); subSetId >= 0; subSetId--)
@@ -2775,6 +2778,12 @@ void CABACWriter::transform_skip_flag( const TransformUnit& tu, ComponentID comp
{
return;
}
+#if JVET_M0140_SBT
+ if( tu.cu->sbtInfo )
+ {
+ return;
+ }
+#endif
m_BinEncoder.encodeBin( tu.transformSkip[compID], Ctx::TransformSkipFlag(toChannelType(compID)) );
DTRACE( g_trace_ctx, D_SYNTAX, "transform_skip_flag() etype=%d pos=(%d,%d) trSkip=%d\n", compID, tu.blocks[compID].x, tu.blocks[compID].y, (int)tu.transformSkip[compID] );
@@ -2809,6 +2818,12 @@ void CABACWriter::emt_cu_flag( const CodingUnit& cu )
{
return;
}
+#endif
+#if JVET_M0140_SBT
+ if( cu.sbtInfo )
+ {
+ return;
+ }
#endif
const CodingStructure& cs = *cu.cs;
diff --git a/source/Lib/EncoderLib/EncCu.cpp b/source/Lib/EncoderLib/EncCu.cpp
index 6f1fea353..0e4108899 100644
--- a/source/Lib/EncoderLib/EncCu.cpp
+++ b/source/Lib/EncoderLib/EncCu.cpp
@@ -718,10 +718,18 @@ void EncCu::xCompressCU( CodingStructure *&tempCS, CodingStructure *&bestCS, Par
bestCS->chType = partitioner.chType;
m_modeCtrl->initCULevel( partitioner, *tempCS );
#if JVET_M0140_SBT
+#if JVET_M0464_UNI_MTS
if( partitioner.currQtDepth == 0 && partitioner.currMtDepth == 0 && !tempCS->slice->isIntra() && ( sps.getUseSBT() || sps.getUseInterMTS() ) )
+#else
+ if( partitioner.currQtDepth == 0 && partitioner.currMtDepth == 0 && !tempCS->slice->isIntra() && ( sps.getUseSBT() || sps.getUseInterEMT() ) )
+#endif
{
auto slsSbt = dynamic_cast<SaveLoadEncInfoSbt*>( m_modeCtrl );
+#if JVET_M0464_UNI_MTS
int maxSLSize = sps.getUseSBT() ? tempCS->slice->getSPS()->getMaxSbtSize() : MTS_INTER_MAX_CU_SIZE;
+#else
+ int maxSLSize = sps.getUseSBT() ? tempCS->slice->getSPS()->getMaxSbtSize() : EMT_INTER_MAX_CU_WITH_QTBT;
+#endif
slsSbt->resetSaveloadSbt( maxSLSize );
}
m_sbtCostSave[0] = m_sbtCostSave[1] = MAX_DOUBLE;
@@ -4237,7 +4245,11 @@ void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&be
}
}
#if JVET_M0140_SBT
+#if JVET_M0464_UNI_MTS
const bool mtsAllowed = tempCS->sps->getUseInterMTS() && partitioner.currArea().lwidth() <= MTS_INTER_MAX_CU_SIZE && partitioner.currArea().lheight() <= MTS_INTER_MAX_CU_SIZE;
+#else
+ const bool mtsAllowed = considerEmtSecondPass;
+#endif
uint8_t sbtAllowed = cu->checkAllowedSbt();
uint8_t numRDOTried = 0;
Distortion sbtOffDist = 0;
@@ -4335,7 +4347,11 @@ void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&be
#endif
m_pcInterSearch->encodeResAndCalcRdInterCU( *tempCS, partitioner, skipResidual );
#if JVET_M0140_SBT
+#if JVET_M0464_UNI_MTS
numRDOTried += mtsAllowed ? 2 : 1;
+#else
+ numRDOTried++;
+#endif
#endif
xEncodeDontSplit( *tempCS, partitioner );
@@ -4400,12 +4416,31 @@ void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&be
}
}
#if JVET_M0140_SBT
+#if JVET_M0464_UNI_MTS
currBestCost = tempCS->cost;
sbtOffCost = tempCS->cost;
sbtOffDist = tempCS->dist;
sbtOffRootCbf = cu->rootCbf;
currBestSbt = CU::getSbtInfo( cu->firstTU->mtsIdx > 1 ? SBT_OFF_MTS : SBT_OFF_DCT, 0 );
currBestTrs = cu->firstTU->mtsIdx;
+#else
+ if( curEmtMode == 0 )
+ {
+ currBestCost = tempCS->cost;
+ sbtOffCost = tempCS->cost;
+ sbtOffDist = tempCS->dist;
+ sbtOffRootCbf = cu->rootCbf;
+ }
+ else
+ {
+ if( tempCS->cost < currBestCost )
+ {
+ currBestSbt = CU::getSbtInfo(SBT_OFF_MTS, 0);
+ currBestTrs = cu->firstTU->emtIdx;
+ currBestCost = tempCS->cost;
+ }
+ }
+#endif
if( cu->lwidth() <= MAX_TU_SIZE_FOR_PROFILE && cu->lheight() <= MAX_TU_SIZE_FOR_PROFILE )
{
CHECK( tempCS->tus.size() != 1, "tu must be only one" );
@@ -4421,7 +4456,11 @@ void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&be
#if !JVET_M0464_UNI_MTS
//now we check whether the second pass should be skipped or not
+#if JVET_M0140_SBT
+ if( !curEmtMode && maxEMTMode && !CU::isMtsMode(histBestSbt) )
+#else
if( !curEmtMode && maxEMTMode )
+#endif
{
const double thresholdToSkipEmtSecondPass = 1.1; // Skip checking EMT transforms
const bool bCond1 = !cu->firstTU->cbf[COMPONENT_Y];
@@ -4439,10 +4478,20 @@ void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&be
#endif
#if JVET_M0140_SBT //RDO for SBT
+#if !JVET_M0464_UNI_MTS // skip EMT
+ if( histBestSbt != MAX_UCHAR && !CU::isMtsMode(histBestSbt) )
+ {
+ maxEMTMode = 0;
+ }
+#endif
uint8_t numSbtRdo = CU::numSbtModeRdo( sbtAllowed );
//early termination if all SBT modes are not allowed
//normative
+#if JVET_M0464_UNI_MTS
if( !sbtAllowed || skipResidual )
+#else
+ if( !sbtAllowed || skipResidual || cu->emtFlag )
+#endif
{
numSbtRdo = 0;
}
@@ -4535,6 +4584,9 @@ void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&be
tempCS->dist = 0;
tempCS->fracBits = 0;
tempCS->cost = MAX_DOUBLE;
+#if !JVET_M0464_UNI_MTS
+ cu->emtFlag = curEmtMode;
+#endif
cu->skip = false;
//set SBT info
@@ -4568,7 +4620,11 @@ void EncCu::xEncodeInterResidual( CodingStructure *&tempCS, CodingStructure *&be
if( tempCS->cost < currBestCost )
{
currBestSbt = cu->sbtInfo;
+#if JVET_M0464_UNI_MTS
currBestTrs = tempCS->tus[cu->sbtInfo ? cu->getSbtPos() : 0]->mtsIdx;
+#else
+ currBestTrs = tempCS->tus[cu->sbtInfo ? cu->getSbtPos() : 0]->emtIdx;
+#endif
assert( currBestTrs == 0 || currBestTrs == 1 );
currBestCost = tempCS->cost;
}
diff --git a/source/Lib/EncoderLib/InterSearch.cpp b/source/Lib/EncoderLib/InterSearch.cpp
index a67a0681d..608e1c001 100644
--- a/source/Lib/EncoderLib/InterSearch.cpp
+++ b/source/Lib/EncoderLib/InterSearch.cpp
@@ -6396,6 +6396,9 @@ void InterSearch::xEstimateInterResidualQT(CodingStructure &cs, Partitioner &par
{
checkTransformSkip[compID] &= !tu.cu->emtFlag;
}
+#if JVET_M0140_SBT
+ checkTransformSkip[compID] &= !tu.cu->sbtInfo;
+#endif
#endif
const bool isCrossCPredictionAvailable = TU::hasCrossCompPredInfo( tu, compID );
@@ -6456,9 +6459,14 @@ void InterSearch::xEstimateInterResidualQT(CodingStructure &cs, Partitioner &par
const int crossCPredictionModesToTest = preCalcAlpha != 0 ? 2 : 1;
#if JVET_M0464_UNI_MTS
const int numTransformCandidates = nNumTransformCands;
+#else
+#if JVET_M0140_SBT
+ const int numEmtTransformCandidates = isLuma(compID) && tu.cu->emtFlag && sps.getUseInterEMT() && (m_histBestMtsIdx == MAX_UCHAR || !sps.getUseSBT()) ? 4 : 1;
+ const int numTransformCandidates = tu.noResidual ? 1 : (checkTransformSkip[compID] ? (numEmtTransformCandidates + 1) : numEmtTransformCandidates);
#else
const int numEmtTransformCandidates = isLuma(compID) && tu.cu->emtFlag && sps.getUseInterEMT() ? 4 : 1;
const int numTransformCandidates = checkTransformSkip[compID] ? ( numEmtTransformCandidates + 1 ) : numEmtTransformCandidates;
+#endif
int lastTransformModeIndex = numTransformCandidates - 1; //lastTransformModeIndex is the mode for transformSkip (if transformSkip is active)
#endif
const bool isOneMode = crossCPredictionModesToTest == 1 && numTransformCandidates == 1;
@@ -6491,8 +6499,12 @@ void InterSearch::xEstimateInterResidualQT(CodingStructure &cs, Partitioner &par
}
tu.mtsIdx = trModes[transformMode].first;
}
+#else
+#if JVET_M0140_SBT
+ if( isLuma( compID ) ) tu.emtIdx = (m_histBestMtsIdx == MAX_UCHAR || !sps.getUseSBT()) ? transformMode : m_histBestMtsIdx;
#else
if( isLuma( compID ) ) tu.emtIdx = transformMode;
+#endif
tu.transformSkip[compID] = checkTransformSkip[compID] && transformMode == lastTransformModeIndex;
#endif
tu.compAlpha[compID] = bUseCrossCPrediction ? preCalcAlpha : 0;
--
GitLab