From 83cad1c88f939a4cc346e0438ec1a12c7e152512 Mon Sep 17 00:00:00 2001
From: Adrian Browne <adrian.browne@sony.com>
Date: Thu, 19 Nov 2020 12:14:02 +0000
Subject: [PATCH] Aligned clipping after inverse transform with VVC
specification
---
source/Lib/CommonLib/TrQuant.cpp | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/source/Lib/CommonLib/TrQuant.cpp b/source/Lib/CommonLib/TrQuant.cpp
index 535a773681..95e339fff0 100644
--- a/source/Lib/CommonLib/TrQuant.cpp
+++ b/source/Lib/CommonLib/TrQuant.cpp
@@ -812,6 +812,8 @@ void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCo
const int TRANSFORM_MATRIX_SHIFT = g_transformMatrixShift[TRANSFORM_INVERSE];
const TCoeff clipMinimum = -( 1 << maxLog2TrDynamicRange );
const TCoeff clipMaximum = ( 1 << maxLog2TrDynamicRange ) - 1;
+ const TCoeff pelMinimum = std::numeric_limits<Pel>::min();
+ const TCoeff pelMaximum = std::numeric_limits<Pel>::max();
const uint32_t transformWidthIndex = floorLog2(width ) - 1; // nLog2WidthMinus1, since transform start from 2-point
const uint32_t transformHeightIndex = floorLog2(height) - 1; // nLog2HeightMinus1, since transform start from 2-point
@@ -844,22 +846,22 @@ void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCo
CHECK( shift_1st < 0, "Negative shift" );
CHECK( shift_2nd < 0, "Negative shift" );
TCoeff *tmp = ( TCoeff * ) alloca( width * height * sizeof( TCoeff ) );
- fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, clipMinimum, clipMaximum);
- fastInvTrans[trTypeHor][transformWidthIndex] (tmp, block, shift_2nd, height, 0, skipWidth, clipMinimum, clipMaximum);
+ fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, clipMinimum, clipMaximum);
+ fastInvTrans[trTypeHor][transformWidthIndex] (tmp, block, shift_2nd, height, 0, skipWidth, pelMinimum, pelMaximum);
}
else if( width == 1 ) //1-D vertical transform
{
int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
CHECK( shift < 0, "Negative shift" );
CHECK( ( transformHeightIndex < 0 ), "There is a problem with the height." );
- fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, clipMinimum, clipMaximum );
+ fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, pelMinimum, pelMaximum );
}
else //if(iHeight == 1) //1-D horizontal transform
{
const int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
CHECK( shift < 0, "Negative shift" );
CHECK( ( transformWidthIndex < 0 ), "There is a problem with the width." );
- fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, clipMinimum, clipMaximum );
+ fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, pelMinimum, pelMaximum );
}
Pel *resiBuf = pResidual.buf;
--
GitLab