From f765becc24b4b8e590960b6dc2d87fd594dd9e86 Mon Sep 17 00:00:00 2001 From: Karl Sharman Date: Fri, 1 Feb 2019 11:27:49 +0000 Subject: [PATCH] Fixed determination of TS mode and fixed typos in base-VTM var names. --- source/Lib/CommonLib/QuantRDOQ.cpp | 16 ++++++---------- source/Lib/CommonLib/UnitTools.cpp | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/source/Lib/CommonLib/QuantRDOQ.cpp b/source/Lib/CommonLib/QuantRDOQ.cpp index 36866aaf8..d711ea75b 100644 --- a/source/Lib/CommonLib/QuantRDOQ.cpp +++ b/source/Lib/CommonLib/QuantRDOQ.cpp @@ -417,16 +417,12 @@ double QuantRDOQ::xGetErrScaleCoeff( SizeType width, SizeType height, int qp, co const int iTransformShift = getTransformShift(channelBitDepth, Size(width, height), maxLog2TrDynamicRange); #if HM_QTBT_AS_IN_JEM_QUANT double dErrScale = (double)( 1 << SCALE_BITS ); // Compensate for scaling of bitcount in Lagrange cost function -#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT +#if !JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT + bool needsSqrt2 = TU::needsBlockSizeTrafoScale( Size(width, height) );// ( ( (sizeX+sizeY) & 1 ) !=0 ); +#endif double dTransShift = (double)iTransformShift + ( needsSqrt2 ? -0.5 : 0.0 ); dErrScale = dErrScale*pow( 2.0, ( -2.0*dTransShift ) ); // Compensate for scaling through forward transform int QStep = ( needsSqrt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] ); -#else - bool needsSrqt2 = TU::needsBlockSizeTrafoScale( Size(width, height) );// ( ( (sizeX+sizeY) & 1 ) !=0 ); - double dTransShift = (double)iTransformShift + ( needsSrqt2 ? -0.5 : 0.0 ); - dErrScale = dErrScale*pow( 2.0, ( -2.0*dTransShift ) ); // Compensate for scaling through forward transform - int QStep = ( needsSrqt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] ); -#endif double finalErrScale = dErrScale / QStep / QStep / (1 << (DISTORTION_PRECISION_ADJUSTMENT(channelBitDepth) << 1)); #else int errShift = SCALE_BITS - ((iTransformShift + DISTORTION_PRECISION_ADJUSTMENT(channelBitDepth)) << 1); @@ -464,8 +460,8 @@ void QuantRDOQ::xSetErrScaleCoeff( uint32_t list, uint32_t sizeX, uint32_t sizeY #if HM_QTBT_AS_IN_JEM_QUANT double dErrScale = (double)( 1 << SCALE_BITS ); // Compensate for scaling of bitcount in Lagrange cost function - bool needsSrqt2 = TU::needsBlockSizeTrafoScale( Size( g_scalingListSizeX[sizeX], g_scalingListSizeX[sizeY] ) );// ( ( (sizeX+sizeY) & 1 ) !=0 ); - double dTransShift = (double)iTransformShift + ( needsSrqt2 ? -0.5 : 0.0 ); + bool needsSqrt2 = TU::needsBlockSizeTrafoScale( Size( g_scalingListSizeX[sizeX], g_scalingListSizeX[sizeY] ) );// ( ( (sizeX+sizeY) & 1 ) !=0 ); + double dTransShift = (double)iTransformShift + ( needsSqrt2 ? -0.5 : 0.0 ); dErrScale = dErrScale*pow( 2.0, ( -2.0*dTransShift ) ); // Compensate for scaling through forward transform for( i = 0; i < uiMaxNumCoeff; i++ ) @@ -474,7 +470,7 @@ void QuantRDOQ::xSetErrScaleCoeff( uint32_t list, uint32_t sizeX, uint32_t sizeY / (1 << (DISTORTION_PRECISION_ADJUSTMENT(bitDepths.recon[channelType]) << 1)); } - int QStep = ( needsSrqt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] ); + int QStep = ( needsSqrt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] ); xGetErrScaleCoeffNoScalingList(list, sizeX, sizeY, qp) = dErrScale / QStep / QStep / (1 << (DISTORTION_PRECISION_ADJUSTMENT(bitDepths.recon[channelType]) << 1)); diff --git a/source/Lib/CommonLib/UnitTools.cpp b/source/Lib/CommonLib/UnitTools.cpp index 5b0fb9381..9b83015aa 100644 --- a/source/Lib/CommonLib/UnitTools.cpp +++ b/source/Lib/CommonLib/UnitTools.cpp @@ -4816,7 +4816,7 @@ bool TU::needsSqrt2Scale( const TransformUnit &tu, const ComponentID &compID ) { const Size &size=tu.blocks[compID]; #if JVET_M0464_UNI_MTS - const bool bTransformSkip = tu.mtsIdx==1; + const bool bTransformSkip = tu.mtsIdx==1 && isLuma(compID); #else const bool bTransformSkip = tu.transformSkip[compID]; #endif -- GitLab