Commit 65a4a484 authored by Karsten Suehring's avatar Karsten Suehring

Merge branch 'JVET-N0847_Scaling_List_v3' into 'master'

Fix for Ticket #311 Scaling List Bugs

See merge request jvet/VVCSoftware_VTM!601
parents 2637b3c2 0a4c160c
Pipeline #1893 passed with stage
......@@ -753,12 +753,10 @@ namespace DQIntern
if (shift < 0)
{
invQScale <<= -shift;
shift = 0;
//add = (1 << shift) >> 1;
}
Intermediate_Int qIdx = ( level << 1 ) + ( level > 0 ? -(state>>1) : (state>>1) );
Intermediate_Int nomTCoeff = ( qIdx * invQScale + add ) >> shift;
tCoeff[ rasterPos ] = (TCoeff)Clip3<Intermediate_Int>( minTCoeff, maxTCoeff, nomTCoeff );
int64_t nomTCoeff = ((int64_t)qIdx * (int64_t)invQScale + add) >> ((shift < 0) ? 0 : shift);
tCoeff[rasterPos] = (TCoeff)Clip3<int64_t>(minTCoeff, maxTCoeff, nomTCoeff);
}
state = ( 32040 >> ((state<<2)+((level&1)<<1)) ) & 3; // the 16-bit value "32040" represent the state transition table
}
......@@ -1615,7 +1613,7 @@ void DepQuant::quant( TransformUnit &tu, const ComponentID &compID, const CCoeff
CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
const uint32_t log2TrWidth = g_aucLog2[width];
const uint32_t log2TrHeight = g_aucLog2[height];
const bool enableScalingLists = getUseScalingList(width, height, tu.mtsIdx == MTS_SKIP);
const bool enableScalingLists = getUseScalingList(width, height, (tu.mtsIdx == MTS_SKIP && isLuma(compID)));
static_cast<DQIntern::DepQuant*>(p)->quant( tu, pSrc, compID, cQP, Quant::m_dLambda, ctx, uiAbsSum, enableScalingLists, Quant::getQuantCoeff(scalingListType, qpRem, log2TrWidth, log2TrHeight) );
}
else
......@@ -1639,7 +1637,7 @@ void DepQuant::dequant( const TransformUnit &tu, CoeffBuf &dstCoeff, const Compo
const uint32_t log2TrWidth = g_aucLog2[width];
const uint32_t log2TrHeight = g_aucLog2[height];
const bool enableScalingLists = getUseScalingList(width, height, (tu.mtsIdx == MTS_SKIP));
const bool enableScalingLists = getUseScalingList(width, height, (tu.mtsIdx == MTS_SKIP && isLuma(compID)));
static_cast<DQIntern::DepQuant*>(p)->dequant( tu, dstCoeff, compID, cQP, enableScalingLists, Quant::getDequantCoeff(scalingListType, qpRem, log2TrWidth, log2TrHeight) );
}
else
......
......@@ -399,7 +399,7 @@ void Quant::dequant(const TransformUnit &tu,
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
int *piDequantCoef = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth - 1, uiLog2TrHeight - 1);
int *piDequantCoef = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth, uiLog2TrHeight);
if(rightShift > 0)
{
......@@ -913,7 +913,7 @@ void Quant::quant(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf
const CCoeffBuf &piCoef = pSrc;
CoeffBuf piQCoef = tu.getCoeffs(compID);
const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
const bool useTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
const int maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange(toChannelType(compID));
{
......@@ -927,7 +927,7 @@ void Quant::quant(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf
CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, useTransformSkip);
......@@ -990,7 +990,7 @@ bool Quant::xNeedRDOQ(TransformUnit &tu, const ComponentID &compID, const CCoeff
const CCoeffBuf piCoef = pSrc;
const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
const bool useTransformSkip = tu.mtsIdx == MTS_SKIP && isLuma(compID);
const int maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange(toChannelType(compID));
int scalingListType = getScalingListType(tu.cu->predMode, compID);
......@@ -998,7 +998,7 @@ bool Quant::xNeedRDOQ(TransformUnit &tu, const ComponentID &compID, const CCoeff
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, (useTransformSkip != 0));
......@@ -1056,7 +1056,7 @@ void Quant::transformSkipQuantOneSample(TransformUnit &tu, const ComponentID &co
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
const int *const piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
const int *const piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
/* for 422 chroma blocks, the effective scaling applied during transformation is not a power of 2, hence it cannot be
* implemented as a bit-shift (the quantised result will be sqrt(2) * larger than required). Alternatively, adjust the
......@@ -1130,7 +1130,7 @@ void Quant::invTrSkipDeQuantOneSample(TransformUnit &tu, const ComponentID &comp
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
int *piDequantCoef = getDequantCoeff(scalingListType,QP_rem,uiLog2TrWidth-1, uiLog2TrHeight-1);
int *piDequantCoef = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth, uiLog2TrHeight);
if (rightShift > 0)
{
......
......@@ -511,7 +511,7 @@ void QuantRDOQ::quant(TransformUnit &tu, const ComponentID &compID, const CCoeff
const CCoeffBuf &piCoef = pSrc;
CoeffBuf piQCoef = tu.getCoeffs(compID);
const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
const bool useTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
bool useRDOQ = useTransformSkip ? m_useRDOQTS : m_useRDOQ;
......@@ -620,8 +620,8 @@ void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID,
const bool needSqrtAdjustment= TU::needsBlockSizeTrafoScale( tu, compID );
const double *const pdErrScale = xGetErrScaleCoeffSL(scalingListType, (uiLog2BlockWidth - 1), (uiLog2BlockHeight - 1), cQP.rem);
const int *const piQCoef = getQuantCoeff(scalingListType, cQP.rem, (uiLog2BlockWidth-1), (uiLog2BlockHeight-1));
const double *const pdErrScale = xGetErrScaleCoeffSL(scalingListType, uiLog2BlockWidth, uiLog2BlockHeight, cQP.rem);
const int *const piQCoef = getQuantCoeff(scalingListType, cQP.rem, uiLog2BlockWidth, uiLog2BlockHeight);
const bool isTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, isTransformSkip);
const int defaultQuantisationCoefficient = g_quantScales[ needSqrtAdjustment ?1:0][cQP.rem];
......
......@@ -1696,7 +1696,7 @@ void ScalingList::checkPredMode(uint32_t sizeId, uint32_t listId)
{
for (int predListIdx = (int)listId; predListIdx >= 0; predListIdx--)
{
if ((sizeId == SCALING_LIST_64x64 && (listId % 3) != 0) || (sizeId == SCALING_LIST_2x2 && (listId % 3) == 0))
if ((sizeId == SCALING_LIST_64x64 && ((listId % 3) != 0 || (predListIdx % 3) != 0)) || (sizeId == SCALING_LIST_2x2 && ((listId % 3) == 0 || (predListIdx % 3) == 0)))
continue;
if( !::memcmp(getScalingListAddress(sizeId,listId),((listId == predListIdx) ?
getScalingListDefaultAddress(sizeId, predListIdx): getScalingListAddress(sizeId, predListIdx)),sizeof(int)*std::min(MAX_MATRIX_COEF_NUM,(int)g_scalingListSize[sizeId])) // check value of matrix
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment