From e276e285f8d97367386d0d7bf4c0d72fe1f10ff7 Mon Sep 17 00:00:00 2001
From: Brian Heng <brian.heng@broadcom.com>
Date: Fri, 6 Dec 2019 14:12:40 -0800
Subject: [PATCH] Fix for Ticket #744 - Overflow in joint Cb/Cr transform
---
source/Lib/CommonLib/TrQuant.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/source/Lib/CommonLib/TrQuant.cpp b/source/Lib/CommonLib/TrQuant.cpp
index e3ebc5e34..3ca2f43c1 100644
--- a/source/Lib/CommonLib/TrQuant.cpp
+++ b/source/Lib/CommonLib/TrQuant.cpp
@@ -149,7 +149,7 @@ template<int signedMode> void invTransformCbCr( PelBuf &resCb, PelBuf &resCr )
if ( signedMode == 1 ) { cr[x] = cb[x] >> 1; }
else if ( signedMode == -1 ) { cr[x] = -cb[x] >> 1; }
else if ( signedMode == 2 ) { cr[x] = cb[x]; }
- else if ( signedMode == -2 ) { cr[x] = -cb[x]; }
+ else if ( signedMode == -2 ) { cr[x] = (cb[x] == -32768 && sizeof(Pel) == 2) ? 32767 : -cb[x]; } // non-normative clipping to prevent 16-bit overflow
else if ( signedMode == 3 ) { cb[x] = cr[x] >> 1; }
else if ( signedMode == -3 ) { cb[x] = -cr[x] >> 1; }
}
--
GitLab