Skip to content
Snippets Groups Projects
Commit 83cad1c8 authored by Adrian Browne's avatar Adrian Browne
Browse files

Aligned clipping after inverse transform with VVC specification

parent a46a3c30
No related branches found
No related tags found
1 merge request!1959Fix #1431: Avoid clipping after inverse transform when RExt__HIGH_BIT_DEPTH_SUPPORT is enabled
......@@ -812,6 +812,8 @@ void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCo
const int TRANSFORM_MATRIX_SHIFT = g_transformMatrixShift[TRANSFORM_INVERSE];
const TCoeff clipMinimum = -( 1 << maxLog2TrDynamicRange );
const TCoeff clipMaximum = ( 1 << maxLog2TrDynamicRange ) - 1;
const TCoeff pelMinimum = std::numeric_limits<Pel>::min();
const TCoeff pelMaximum = std::numeric_limits<Pel>::max();
const uint32_t transformWidthIndex = floorLog2(width ) - 1; // nLog2WidthMinus1, since transform start from 2-point
const uint32_t transformHeightIndex = floorLog2(height) - 1; // nLog2HeightMinus1, since transform start from 2-point
......@@ -844,22 +846,22 @@ void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCo
CHECK( shift_1st < 0, "Negative shift" );
CHECK( shift_2nd < 0, "Negative shift" );
TCoeff *tmp = ( TCoeff * ) alloca( width * height * sizeof( TCoeff ) );
fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, clipMinimum, clipMaximum);
fastInvTrans[trTypeHor][transformWidthIndex] (tmp, block, shift_2nd, height, 0, skipWidth, clipMinimum, clipMaximum);
fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, clipMinimum, clipMaximum);
fastInvTrans[trTypeHor][transformWidthIndex] (tmp, block, shift_2nd, height, 0, skipWidth, pelMinimum, pelMaximum);
}
else if( width == 1 ) //1-D vertical transform
{
int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
CHECK( shift < 0, "Negative shift" );
CHECK( ( transformHeightIndex < 0 ), "There is a problem with the height." );
fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, clipMinimum, clipMaximum );
fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, pelMinimum, pelMaximum );
}
else //if(iHeight == 1) //1-D horizontal transform
{
const int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
CHECK( shift < 0, "Negative shift" );
CHECK( ( transformWidthIndex < 0 ), "There is a problem with the width." );
fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, clipMinimum, clipMaximum );
fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, pelMinimum, pelMaximum );
}
Pel *resiBuf = pResidual.buf;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment