Commit 69b2d08f authored by Frank Bossen's avatar Frank Bossen

Merge branch 'fix_ticket_363' into 'master'

Fix for Ticket #363 Multiplication Overflow in ALF Classification

See merge request !679
parents 64b42ba3 76c454e3
Pipeline #1912 passed with stage
......@@ -843,7 +843,7 @@ void AdaptiveLoopFilter::deriveClassificationBlk(AlfClassifier** classifier, int
d0 = sumD0;
dirTempD = 2;
}
if( d1*hv0 > hv1*d0 )
if( (uint32_t)d1 * (uint32_t)hv0 > (uint32_t)hv1 * (uint32_t)d0 )
{
hvd1 = d1;
hvd0 = d0;
......
......@@ -276,7 +276,7 @@ static void simdDeriveClassificationBlk(AlfClassifier** classifier, int** laplac
xmm6 = _mm_mullo_epi32( xmm8, xmm6 );
xmm9 = _mm_shuffle_epi32( xmm6, 0xB1 );
xmm5 = _mm_cmpgt_epi32( xmm6, xmm9 );
xmm5 = _mm_cmpgt_epi32(_mm_add_epi32(xmm6, _mm_set1_epi32(0x80000000)), _mm_add_epi32(xmm9, _mm_set1_epi32(0x80000000)));
xmm5 = _mm_shuffle_epi32( xmm5, 0xF0 ); //second mask is for all upper part
xmm8 = _mm_shuffle_epi32( xmm4, 0x0E );
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment