Commit f765becc authored by Karl Sharman's avatar Karl Sharman

Fixed determination of TS mode and fixed typos in base-VTM var names.

parent e19fd4c9
......@@ -417,16 +417,12 @@ double QuantRDOQ::xGetErrScaleCoeff( SizeType width, SizeType height, int qp, co
const int iTransformShift = getTransformShift(channelBitDepth, Size(width, height), maxLog2TrDynamicRange);
#if HM_QTBT_AS_IN_JEM_QUANT
double dErrScale = (double)( 1 << SCALE_BITS ); // Compensate for scaling of bitcount in Lagrange cost function
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
#if !JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
bool needsSqrt2 = TU::needsBlockSizeTrafoScale( Size(width, height) );// ( ( (sizeX+sizeY) & 1 ) !=0 );
#endif
double dTransShift = (double)iTransformShift + ( needsSqrt2 ? -0.5 : 0.0 );
dErrScale = dErrScale*pow( 2.0, ( -2.0*dTransShift ) ); // Compensate for scaling through forward transform
int QStep = ( needsSqrt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] );
#else
bool needsSrqt2 = TU::needsBlockSizeTrafoScale( Size(width, height) );// ( ( (sizeX+sizeY) & 1 ) !=0 );
double dTransShift = (double)iTransformShift + ( needsSrqt2 ? -0.5 : 0.0 );
dErrScale = dErrScale*pow( 2.0, ( -2.0*dTransShift ) ); // Compensate for scaling through forward transform
int QStep = ( needsSrqt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] );
#endif
double finalErrScale = dErrScale / QStep / QStep / (1 << (DISTORTION_PRECISION_ADJUSTMENT(channelBitDepth) << 1));
#else
int errShift = SCALE_BITS - ((iTransformShift + DISTORTION_PRECISION_ADJUSTMENT(channelBitDepth)) << 1);
......@@ -464,8 +460,8 @@ void QuantRDOQ::xSetErrScaleCoeff( uint32_t list, uint32_t sizeX, uint32_t sizeY
#if HM_QTBT_AS_IN_JEM_QUANT
double dErrScale = (double)( 1 << SCALE_BITS ); // Compensate for scaling of bitcount in Lagrange cost function
bool needsSrqt2 = TU::needsBlockSizeTrafoScale( Size( g_scalingListSizeX[sizeX], g_scalingListSizeX[sizeY] ) );// ( ( (sizeX+sizeY) & 1 ) !=0 );
double dTransShift = (double)iTransformShift + ( needsSrqt2 ? -0.5 : 0.0 );
bool needsSqrt2 = TU::needsBlockSizeTrafoScale( Size( g_scalingListSizeX[sizeX], g_scalingListSizeX[sizeY] ) );// ( ( (sizeX+sizeY) & 1 ) !=0 );
double dTransShift = (double)iTransformShift + ( needsSqrt2 ? -0.5 : 0.0 );
dErrScale = dErrScale*pow( 2.0, ( -2.0*dTransShift ) ); // Compensate for scaling through forward transform
for( i = 0; i < uiMaxNumCoeff; i++ )
......@@ -474,7 +470,7 @@ void QuantRDOQ::xSetErrScaleCoeff( uint32_t list, uint32_t sizeX, uint32_t sizeY
/ (1 << (DISTORTION_PRECISION_ADJUSTMENT(bitDepths.recon[channelType]) << 1));
}
int QStep = ( needsSrqt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] );
int QStep = ( needsSqrt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] );
xGetErrScaleCoeffNoScalingList(list, sizeX, sizeY, qp) =
dErrScale / QStep / QStep / (1 << (DISTORTION_PRECISION_ADJUSTMENT(bitDepths.recon[channelType]) << 1));
......
......@@ -4816,7 +4816,7 @@ bool TU::needsSqrt2Scale( const TransformUnit &tu, const ComponentID &compID )
{
const Size &size=tu.blocks[compID];
#if JVET_M0464_UNI_MTS
const bool bTransformSkip = tu.mtsIdx==1;
const bool bTransformSkip = tu.mtsIdx==1 && isLuma(compID);
#else
const bool bTransformSkip = tu.transformSkip[compID];
#endif
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment