Commit a01f8a51 authored by Brian Heng's avatar Brian Heng

Fix for Ticket #363 Multiplication Overflow in ALF Classification

  - Multiplication of "d" and "hv" terms overlows signed 32-bit data type.
parent bfe158b0
......@@ -843,7 +843,7 @@ void AdaptiveLoopFilter::deriveClassificationBlk(AlfClassifier** classifier, int
d0 = sumD0;
dirTempD = 2;
}
if( d1*hv0 > hv1*d0 )
if( (uint32_t)d1 * (uint32_t)hv0 > (uint32_t)hv1 * (uint32_t)d0 )
{
hvd1 = d1;
hvd0 = d0;
......
......@@ -276,7 +276,7 @@ static void simdDeriveClassificationBlk(AlfClassifier** classifier, int** laplac
xmm6 = _mm_mullo_epi32( xmm8, xmm6 );
xmm9 = _mm_shuffle_epi32( xmm6, 0xB1 );
xmm5 = _mm_cmpgt_epi32( xmm6, xmm9 );
xmm5 = _mm_xor_si128(_mm_cmpeq_epi32(_mm_max_epu32(xmm9, xmm6), xmm9), _mm_set1_epi32(-1));
xmm5 = _mm_shuffle_epi32( xmm5, 0xF0 ); //second mask is for all upper part
xmm8 = _mm_shuffle_epi32( xmm4, 0x0E );
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment