diff --git a/source/Lib/CommonLib/TrQuant.cpp b/source/Lib/CommonLib/TrQuant.cpp index 535a7736818b846f70365e9d12a33b76fd6923ad..95e339fff0df4479f975aa8fd7ce52af541afe61 100644 --- a/source/Lib/CommonLib/TrQuant.cpp +++ b/source/Lib/CommonLib/TrQuant.cpp @@ -812,6 +812,8 @@ void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCo const int TRANSFORM_MATRIX_SHIFT = g_transformMatrixShift[TRANSFORM_INVERSE]; const TCoeff clipMinimum = -( 1 << maxLog2TrDynamicRange ); const TCoeff clipMaximum = ( 1 << maxLog2TrDynamicRange ) - 1; + const TCoeff pelMinimum = std::numeric_limits<Pel>::min(); + const TCoeff pelMaximum = std::numeric_limits<Pel>::max(); const uint32_t transformWidthIndex = floorLog2(width ) - 1; // nLog2WidthMinus1, since transform start from 2-point const uint32_t transformHeightIndex = floorLog2(height) - 1; // nLog2HeightMinus1, since transform start from 2-point @@ -844,22 +846,22 @@ void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCo CHECK( shift_1st < 0, "Negative shift" ); CHECK( shift_2nd < 0, "Negative shift" ); TCoeff *tmp = ( TCoeff * ) alloca( width * height * sizeof( TCoeff ) ); - fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, clipMinimum, clipMaximum); - fastInvTrans[trTypeHor][transformWidthIndex] (tmp, block, shift_2nd, height, 0, skipWidth, clipMinimum, clipMaximum); + fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, clipMinimum, clipMaximum); + fastInvTrans[trTypeHor][transformWidthIndex] (tmp, block, shift_2nd, height, 0, skipWidth, pelMinimum, pelMaximum); } else if( width == 1 ) //1-D vertical transform { int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC; CHECK( shift < 0, "Negative shift" ); CHECK( ( transformHeightIndex < 0 ), "There is a problem with the height." ); - fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, clipMinimum, clipMaximum ); + fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, pelMinimum, pelMaximum ); } else //if(iHeight == 1) //1-D horizontal transform { const int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC; CHECK( shift < 0, "Negative shift" ); CHECK( ( transformWidthIndex < 0 ), "There is a problem with the width." ); - fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, clipMinimum, clipMaximum ); + fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, pelMinimum, pelMaximum ); } Pel *resiBuf = pResidual.buf;