From 80100d47540f2f88913693e87b18224ee9fc3174 Mon Sep 17 00:00:00 2001
From: Brian Heng <brian.heng@broadcom.com>
Date: Fri, 6 Dec 2019 15:37:04 -0800
Subject: [PATCH] Fix for Ticket #687 - Wrap around in scaling list coding.
- Prevent negative coefficients on the decode side.
- Prevent illegal coefficients outside the range [-128,127] on the encode side.
---
source/Lib/DecoderLib/VLCReader.cpp | 4 ++--
source/Lib/EncoderLib/VLCWriter.cpp | 2 ++
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/source/Lib/DecoderLib/VLCReader.cpp b/source/Lib/DecoderLib/VLCReader.cpp
index 78e511b3a..00d56c245 100644
--- a/source/Lib/DecoderLib/VLCReader.cpp
+++ b/source/Lib/DecoderLib/VLCReader.cpp
@@ -4024,7 +4024,7 @@ void HLSyntaxReader::decodeScalingList(ScalingList *scalingList, uint32_t sizeId
{
predCoef = (PredListId >= SCALING_LIST_1D_START_16x16) ? scalingList->getScalingListDC(PredListId) : srcPred[0];
}
- scalingList->setScalingListDC(scalingListId, (nextCoef + predCoef + 256) % 256);
+ scalingList->setScalingListDC(scalingListId, (nextCoef + predCoef + 256) & 255);
#else
int nextCoef = SCALING_LIST_START_VALUE;
ScanElement *scan = g_scanOrder[SCAN_UNGROUPED][SCAN_DIAG][gp_sizeIdxInfo->idxFrom(1 << (sizeId == SCALING_LIST_2x2 ? 1 : (sizeId == SCALING_LIST_4x4 ? 2 : 3)))][gp_sizeIdxInfo->idxFrom(1 << (sizeId == SCALING_LIST_2x2 ? 1 : (sizeId == SCALING_LIST_4x4 ? 2 : 3)))];
@@ -4053,7 +4053,7 @@ void HLSyntaxReader::decodeScalingList(ScalingList *scalingList, uint32_t sizeId
#if JVET_P01034_PRED_1D_SCALING_LIST
nextCoef += data;
predCoef = (isPredictor) ? srcPred[scan[i].idx] : 0;
- dst[scan[i].idx] = (nextCoef + predCoef + 256) % 256;
+ dst[scan[i].idx] = (nextCoef + predCoef + 256) & 255;
#else
nextCoef = (nextCoef + data + 256) % 256;
dst[scan[i].idx] = nextCoef;
diff --git a/source/Lib/EncoderLib/VLCWriter.cpp b/source/Lib/EncoderLib/VLCWriter.cpp
index 988167e56..8ba07f50c 100644
--- a/source/Lib/EncoderLib/VLCWriter.cpp
+++ b/source/Lib/EncoderLib/VLCWriter.cpp
@@ -2680,6 +2680,7 @@ void HLSWriter::xCodeScalingList(const ScalingList* scalingList, uint32_t sizeId
data = scalingList->getScalingListDC(scalingListId) - nextCoef;
nextCoef = scalingList->getScalingListDC(scalingListId);
}
+ data = ((data + 128) & 255) - 128;
WRITE_SVLC((int8_t)data, "scaling_list_dc_coef");
#else
const int *src = scalingList->getScalingListAddress(sizeId, listId);
@@ -2700,6 +2701,7 @@ void HLSWriter::xCodeScalingList(const ScalingList* scalingList, uint32_t sizeId
#if JVET_P01034_PRED_1D_SCALING_LIST
data = (isPredictor) ? (deltasrc[i] - nextCoef) : (src[scan[i].idx] - nextCoef);
nextCoef = (isPredictor) ? deltasrc[i] : src[scan[i].idx];
+ data = ((data + 128) & 255) - 128;
WRITE_SVLC((int8_t)data, "scaling_list_delta_coef");
#else
data = src[scan[i].idx] - nextCoef;
--
GitLab