Commit 59d9c06c authored by Frank Bossen's avatar Frank Bossen

Merge branch 'M0119-RemoveScalingOfTSCoefficients' into 'master'

M0119 remove scaling of ts coefficients

See merge request !218
parents e9a4a4ff a772f856
Pipeline #703 passed with stage
......@@ -666,8 +666,13 @@ namespace DQIntern
m_QShift = QUANT_SHIFT - 1 + qpPer + transformShift;
m_QAdd = -( ( 3 << m_QShift ) >> 1 );
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
Intermediate_Int invShift = IQUANT_SHIFT + 1 - qpPer - transformShift + ( TU::needsBlockSizeTrafoScale( tu, compID ) ? ADJ_DEQUANT_SHIFT : 0 );
m_QScale = ( TU::needsSqrt2Scale( tu, compID ) ? ( g_quantScales[ qpRem ] * 181 ) >> 7 : g_quantScales[ qpRem ] );
#else
Intermediate_Int invShift = IQUANT_SHIFT + 1 - qpPer - transformShift + ( TU::needsBlockSizeTrafoScale( area ) ? ADJ_DEQUANT_SHIFT : 0 );
m_QScale = ( TU::needsSqrt2Scale( area ) ? ( g_quantScales[ qpRem ] * 181 ) >> 7 : g_quantScales[ qpRem ] );
#endif
#else
Intermediate_Int invShift = IQUANT_SHIFT + 1 - qpPer - transformShift;
m_QScale = g_quantScales [ qpRem ];
......@@ -750,8 +755,13 @@ namespace DQIntern
#endif
const int transformShift = ( clipTransformShift ? std::max<int>( 0, nomTransformShift ) : nomTransformShift );
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
Intermediate_Int shift = IQUANT_SHIFT + 1 - qpPer - transformShift + ( TU::needsBlockSizeTrafoScale( tu, compID ) ? ADJ_DEQUANT_SHIFT : 0 );
Intermediate_Int invQScale = g_invQuantScales[ qpRem ] * ( TU::needsSqrt2Scale( tu, compID ) ? 181 : 1 );
#else
Intermediate_Int shift = IQUANT_SHIFT + 1 - qpPer - transformShift + ( TU::needsBlockSizeTrafoScale( area ) ? ADJ_DEQUANT_SHIFT : 0 );
Intermediate_Int invQScale = g_invQuantScales[ qpRem ] * ( TU::needsSqrt2Scale( area ) ? 181 : 1 );
#endif
#else
Intermediate_Int shift = IQUANT_SHIFT + 1 - qpPer - transformShift;
Intermediate_Int invQScale = g_invQuantScales[ qpRem ];
......
......@@ -313,8 +313,13 @@ void Quant::dequant(const TransformUnit &tu,
const int QP_rem = cQP.rem;
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
const bool needsScalingCorrection = TU::needsBlockSizeTrafoScale( tu, compID );
const int NEScale = TU::needsSqrt2Scale( tu, compID ) ? 181 : 1;
#else
const bool needsScalingCorrection = TU::needsBlockSizeTrafoScale( tu.block( compID ) );
const int NEScale = TU::needsSqrt2Scale( tu.blocks[compID] ) ? 181 : 1;
#endif
#if HEVC_USE_SCALING_LISTS
const int rightShift = (needsScalingCorrection ? 8 : 0 ) + (IQUANT_SHIFT - (iTransformShift + QP_per)) + (enableScalingLists ? LOG2_SCALING_LIST_NEUTRAL_VALUE : 0);
#else
......@@ -767,7 +772,11 @@ void Quant::quant(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf
int iWHScale = 1;
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
if( TU::needsBlockSizeTrafoScale( tu, compID ) )
#else
if( TU::needsBlockSizeTrafoScale( rect ) )
#endif
{
iTransformShift += ADJ_QUANT_SHIFT;
iWHScale = 181;
......@@ -863,7 +872,11 @@ bool Quant::xNeedRDOQ(TransformUnit &tu, const ComponentID &compID, const CCoeff
int iWHScale = 1;
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
if( TU::needsBlockSizeTrafoScale( tu, compID ) )
#else
if( TU::needsBlockSizeTrafoScale( rect ) )
#endif
{
iTransformShift += ADJ_QUANT_SHIFT;
iWHScale = 181;
......
......@@ -408,15 +408,21 @@ void QuantRDOQ::setScalingList(ScalingList *scalingList, const int maxLog2TrDyna
#endif
#else
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
double QuantRDOQ::xGetErrScaleCoeff( const bool needsSqrt2, SizeType width, SizeType height, int qp, const int maxLog2TrDynamicRange, const int channelBitDepth )
#else
double QuantRDOQ::xGetErrScaleCoeff( SizeType width, SizeType height, int qp, const int maxLog2TrDynamicRange, const int channelBitDepth )
#endif
{
const int iTransformShift = getTransformShift(channelBitDepth, Size(width, height), maxLog2TrDynamicRange);
#if HM_QTBT_AS_IN_JEM_QUANT
double dErrScale = (double)( 1 << SCALE_BITS ); // Compensate for scaling of bitcount in Lagrange cost function
bool needsSrqt2 = TU::needsBlockSizeTrafoScale( Size(width, height) );// ( ( (sizeX+sizeY) & 1 ) !=0 );
double dTransShift = (double)iTransformShift + ( needsSrqt2 ? -0.5 : 0.0 );
#if !JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
bool needsSqrt2 = TU::needsBlockSizeTrafoScale( Size(width, height) );// ( ( (sizeX+sizeY) & 1 ) !=0 );
#endif
double dTransShift = (double)iTransformShift + ( needsSqrt2 ? -0.5 : 0.0 );
dErrScale = dErrScale*pow( 2.0, ( -2.0*dTransShift ) ); // Compensate for scaling through forward transform
int QStep = ( needsSrqt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] );
int QStep = ( needsSqrt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] );
double finalErrScale = dErrScale / QStep / QStep / (1 << (DISTORTION_PRECISION_ADJUSTMENT(channelBitDepth) << 1));
#else
int errShift = SCALE_BITS - ((iTransformShift + DISTORTION_PRECISION_ADJUSTMENT(channelBitDepth)) << 1);
......@@ -454,8 +460,8 @@ void QuantRDOQ::xSetErrScaleCoeff( uint32_t list, uint32_t sizeX, uint32_t sizeY
#if HM_QTBT_AS_IN_JEM_QUANT
double dErrScale = (double)( 1 << SCALE_BITS ); // Compensate for scaling of bitcount in Lagrange cost function
bool needsSrqt2 = TU::needsBlockSizeTrafoScale( Size( g_scalingListSizeX[sizeX], g_scalingListSizeX[sizeY] ) );// ( ( (sizeX+sizeY) & 1 ) !=0 );
double dTransShift = (double)iTransformShift + ( needsSrqt2 ? -0.5 : 0.0 );
bool needsSqrt2 = TU::needsBlockSizeTrafoScale( Size( g_scalingListSizeX[sizeX], g_scalingListSizeX[sizeY] ) );// ( ( (sizeX+sizeY) & 1 ) !=0 );
double dTransShift = (double)iTransformShift + ( needsSqrt2 ? -0.5 : 0.0 );
dErrScale = dErrScale*pow( 2.0, ( -2.0*dTransShift ) ); // Compensate for scaling through forward transform
for( i = 0; i < uiMaxNumCoeff; i++ )
......@@ -464,7 +470,7 @@ void QuantRDOQ::xSetErrScaleCoeff( uint32_t list, uint32_t sizeX, uint32_t sizeY
/ (1 << (DISTORTION_PRECISION_ADJUSTMENT(bitDepths.recon[channelType]) << 1));
}
int QStep = ( needsSrqt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] );
int QStep = ( needsSqrt2 ? ( ( g_quantScales[qp] * 181 ) >> 7 ) : g_quantScales[qp] );
xGetErrScaleCoeffNoScalingList(list, sizeX, sizeY, qp) =
dErrScale / QStep / QStep / (1 << (DISTORTION_PRECISION_ADJUSTMENT(bitDepths.recon[channelType]) << 1));
......@@ -682,8 +688,13 @@ void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID,
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, tu.transformSkip[compID]);
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
const int defaultQuantisationCoefficient = ( TU::needsSqrt2Scale( rect, tu.transformSkip[compID] ) ? ( g_quantScales[cQP.rem] * 181 ) >> 7 : g_quantScales[cQP.rem] );
const double defaultErrorScale = xGetErrScaleCoeffNoScalingList(scalingListType, (uiLog2BlockWidth-1), (uiLog2BlockHeight-1), cQP.rem);
#else
const int defaultQuantisationCoefficient = ( TU::needsSqrt2Scale( rect ) ? ( g_quantScales[cQP.rem] * 181 ) >> 7 : g_quantScales[cQP.rem] );
const double defaultErrorScale = xGetErrScaleCoeffNoScalingList(scalingListType, (uiLog2BlockWidth-1), (uiLog2BlockHeight-1), cQP.rem);
#endif
#else
const double blkErrScale = ( TU::needsQP3Offset( tu, compID ) ? 2.0 : 1.0 );
const int defaultQuantisationCoefficient = g_quantScales[cQP.rem];
......@@ -691,8 +702,13 @@ void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID,
#endif
#else //HEVC_USE_SCALING_LISTS
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
const int quantisationCoefficient = ( TU::needsSqrt2Scale( tu, compID ) ? ( g_quantScales[cQP.rem] * 181 ) >> 7 : g_quantScales[cQP.rem] );
const double errorScale = xGetErrScaleCoeff( TU::needsSqrt2Scale( tu, compID ), uiWidth, uiHeight, cQP.rem, maxLog2TrDynamicRange, channelBitDepth );
#else
const int quantisationCoefficient = ( TU::needsSqrt2Scale( rect ) ? ( g_quantScales[cQP.rem] * 181 ) >> 7 : g_quantScales[cQP.rem] );
const double errorScale = xGetErrScaleCoeff( uiWidth, uiHeight, cQP.rem, maxLog2TrDynamicRange, channelBitDepth );
#endif
#else
const double blkErrScale = ( TU::needsQP3Offset( tu, compID ) ? 2.0 : 1.0 );
const int quantisationCoefficient = g_quantScales[cQP.rem];
......
......@@ -75,8 +75,12 @@ private:
void xInitScalingList ( const QuantRDOQ* other );
void xDestroyScalingList ();
void xSetErrScaleCoeff ( uint32_t list, uint32_t sizeX, uint32_t sizeY, int qp, const int maxLog2TrDynamicRange[MAX_NUM_CHANNEL_TYPE], const BitDepths &bitDepths );
#else
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
double xGetErrScaleCoeff ( const bool needsSqrt2, SizeType width, SizeType height, int qp, const int maxLog2TrDynamicRange, const int channelBitDepth);
#else
double xGetErrScaleCoeff ( SizeType width, SizeType height, int qp, const int maxLog2TrDynamicRange, const int channelBitDepth);
#endif
#endif
// RDOQ functions
......
......@@ -454,7 +454,7 @@ void TrQuant::xITransformSkip(const CCoeffBuf &pCoeff,
}
int iWHScale = 1;
#if HM_QTBT_AS_IN_JEM_QUANT
#if HM_QTBT_AS_IN_JEM_QUANT && !JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
if( TU::needsBlockSizeTrafoScale( area ) )
{
iTransformShift += ADJ_QUANT_SHIFT;
......@@ -532,7 +532,16 @@ void TrQuant::transformNxN(TransformUnit &tu, const ComponentID &compID, const Q
sumAbs += abs( tempCoeff.buf[pos] );
}
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
double scaleSAD=1.0;
if (isLuma(compID) && tu.mtsIdx==1 && ((g_aucLog2[width] + g_aucLog2[height]) & 1) == 1 )
{
scaleSAD=1.0/1.414213562; // compensate for not scaling transform skip coefficients by 1/sqrt(2)
}
trCosts.push_back( TrCost( int(sumAbs*scaleSAD), pos++ ) );
#else
trCosts.push_back( TrCost( sumAbs, pos++ ) );
#endif
it++;
}
......@@ -781,7 +790,7 @@ void TrQuant::xTransformSkip(const TransformUnit &tu, const ComponentID &compID,
}
int iWHScale = 1;
#if HM_QTBT_AS_IN_JEM_QUANT
#if HM_QTBT_AS_IN_JEM_QUANT && !JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
if( TU::needsBlockSizeTrafoScale( rect ) )
{
iTransformShift -= ADJ_DEQUANT_SHIFT;
......
......@@ -236,6 +236,8 @@ typedef std::pair<int, int> TrCost;
#define HM_JEM_CLIP_PEL 1 // ***
#define HM_JEM_MERGE_CANDS 0 // ***
#define JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT 1 ///< When 1, do not scale transform skip blocks by sqrt(2) between (I)Transform and (I)Quantizer
#endif//JEM_COMP
// ====================================================================================================================
......
......@@ -4811,17 +4811,37 @@ uint32_t TU::getNumNonZeroCoeffsNonTS( const TransformUnit& tu, const bool bLuma
return count;
}
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
bool TU::needsSqrt2Scale( const TransformUnit &tu, const ComponentID &compID )
{
const Size &size=tu.blocks[compID];
#if JVET_M0464_UNI_MTS
const bool isTransformSkip = tu.mtsIdx==1 && isLuma(compID);
#else
const bool isTransformSkip = tu.transformSkip[compID];
#endif
return (!isTransformSkip) && (((g_aucLog2[size.width] + g_aucLog2[size.height]) & 1) == 1);
}
#else
bool TU::needsSqrt2Scale( const Size& size )
{
return (((g_aucLog2[size.width] + g_aucLog2[size.height]) & 1) == 1);
}
#endif
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
bool TU::needsBlockSizeTrafoScale( const TransformUnit &tu, const ComponentID &compID )
{
return needsSqrt2Scale( tu, compID ) || isNonLog2BlockSize( tu.blocks[compID] );
}
#else
bool TU::needsBlockSizeTrafoScale( const Size& size )
{
return needsSqrt2Scale( size ) || isNonLog2BlockSize( size );
}
#endif
#else
bool TU::needsQP3Offset(const TransformUnit &tu, const ComponentID &compID)
{
......
......@@ -200,9 +200,18 @@ namespace TU
#endif
bool hasCrossCompPredInfo (const TransformUnit &tu, const ComponentID &compID);
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
bool needsSqrt2Scale ( const TransformUnit &tu, const ComponentID &compID );
#else
bool needsSqrt2Scale ( const Size& size );
#endif
#if HM_QTBT_AS_IN_JEM_QUANT
#if JVET_M0119_NO_TRANSFORM_SKIP_QUANTISATION_ADJUSTMENT
bool needsBlockSizeTrafoScale ( const TransformUnit &tu, const ComponentID &compID );
#else
bool needsBlockSizeTrafoScale ( const Size& size );
#endif
#else
bool needsQP3Offset (const TransformUnit &tu, const ComponentID &compID);
#endif
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment