From 7c89a371b1d372fec5b90dc8e10854512003614f Mon Sep 17 00:00:00 2001
From: Frank Bossen <fbossen@gmail.com>
Date: Sat, 9 May 2020 12:12:36 -0400
Subject: [PATCH] Remove bad cast from short * to int *
Consistently use 16-bit integers to hold CC-ALF coefficients to
avoid type conversions
---
source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp | 13 ++++++++-----
source/Lib/EncoderLib/EncAdaptiveLoopFilter.h | 4 ++--
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp b/source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp
index 76ce5f0da..d5876a6f6 100644
--- a/source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp
+++ b/source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp
@@ -253,7 +253,7 @@ double AlfCovariance::calcErrorForCoeffs( const int *clip, const int *coeff, con
return error / factor;
}
-double AlfCovariance::calcErrorForCcAlfCoeffs(const int* coeff, const int numCoeff, const int bitDepth) const
+double AlfCovariance::calcErrorForCcAlfCoeffs(const int16_t *coeff, const int numCoeff, const int bitDepth) const
{
double factor = 1 << (bitDepth - 1);
double error = 0;
@@ -1883,7 +1883,8 @@ void EncAdaptiveLoopFilter::roundFiltCoeff( int *filterCoeffQuant, double *filte
}
}
-void EncAdaptiveLoopFilter::roundFiltCoeffCCALF( int *filterCoeffQuant, double *filterCoeff, const int numCoeff, const int factor )
+void EncAdaptiveLoopFilter::roundFiltCoeffCCALF(int16_t *filterCoeffQuant, double *filterCoeff, const int numCoeff,
+ const int factor)
{
for( int i = 0; i < numCoeff; i++ )
{
@@ -3308,7 +3309,7 @@ void EncAdaptiveLoopFilter::deriveCcAlfFilterCoeff( ComponentID compID, const Pe
using Ty = double[MAX_NUM_ALF_LUMA_COEFF];
double filterCoeffDbl[MAX_NUM_CC_ALF_CHROMA_COEFF];
- int filterCoeffInt[MAX_NUM_CC_ALF_CHROMA_COEFF];
+ int16_t filterCoeffInt[MAX_NUM_CC_ALF_CHROMA_COEFF];
std::fill_n(filterCoeffInt, MAX_NUM_CC_ALF_CHROMA_COEFF, 0);
@@ -3836,8 +3837,10 @@ void EncAdaptiveLoopFilter::deriveCcAlfFilter( CodingStructure& cs, ComponentID
for (int x = 0; x < m_buf->width; x += (1 << log2BlockWidth))
{
int ctuIdx = (y >> log2BlockHeight) * m_numCTUsInWidth + (x >> log2BlockWidth);
- m_trainingDistortion[filterIdx][ctuIdx] = int(m_ctbDistortionUnfilter[compID][ctuIdx] +
- m_alfCovarianceCcAlf[compID - 1][0][0][ctuIdx].calcErrorForCcAlfCoeffs((int*)(ccAlfFilterCoeff[filterIdx]), numCoeff, (m_scaleBits + 1)));
+ m_trainingDistortion[filterIdx][ctuIdx] =
+ int(m_ctbDistortionUnfilter[compID][ctuIdx]
+ + m_alfCovarianceCcAlf[compID - 1][0][0][ctuIdx].calcErrorForCcAlfCoeffs(
+ ccAlfFilterCoeff[filterIdx], numCoeff, m_scaleBits + 1));
}
}
#else
diff --git a/source/Lib/EncoderLib/EncAdaptiveLoopFilter.h b/source/Lib/EncoderLib/EncAdaptiveLoopFilter.h
index cecff4c84..0aa905998 100644
--- a/source/Lib/EncoderLib/EncAdaptiveLoopFilter.h
+++ b/source/Lib/EncoderLib/EncAdaptiveLoopFilter.h
@@ -203,7 +203,7 @@ struct AlfCovariance
double calculateError( const int *clip, const double *coeff ) const { return calculateError(clip, coeff, numCoeff); }
double calculateError( const int *clip, const double *coeff, const int numCoeff ) const;
double calcErrorForCoeffs( const int *clip, const int *coeff, const int numCoeff, const int bitDepth ) const;
- double calcErrorForCcAlfCoeffs(const int* coeff, const int numCoeff, const int bitDepth) const;
+ double calcErrorForCcAlfCoeffs(const int16_t *coeff, const int numCoeff, const int bitDepth) const;
void getClipMax(const AlfFilterShape& alfShape, int *clip_max) const;
void reduceClipCost(const AlfFilterShape& alfShape, int *clip) const;
@@ -338,7 +338,7 @@ private:
#endif
const int numClasses, const int numCoeff, double& distUnfilter );
void roundFiltCoeff( int *filterCoeffQuant, double *filterCoeff, const int numCoeff, const int factor );
- void roundFiltCoeffCCALF( int *filterCoeffQuant, double *filterCoeff, const int numCoeff, const int factor );
+ void roundFiltCoeffCCALF(int16_t *filterCoeffQuant, double *filterCoeff, const int numCoeff, const int factor);
double getDistCoeffForce0( bool* codedVarBins, double errorForce0CoeffTab[MAX_NUM_ALF_CLASSES][2], int* bitsVarBin, int zeroBitsVarBin, const int numFilters);
int lengthUvlc( int uiCode );
--
GitLab