diff --git a/source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp b/source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp index 76ce5f0da3a70b0ca590d517ab53bccf7bce89cd..d5876a6f687d204b83a5d82d0c8aca847a318974 100644 --- a/source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp +++ b/source/Lib/EncoderLib/EncAdaptiveLoopFilter.cpp @@ -253,7 +253,7 @@ double AlfCovariance::calcErrorForCoeffs( const int *clip, const int *coeff, con return error / factor; } -double AlfCovariance::calcErrorForCcAlfCoeffs(const int* coeff, const int numCoeff, const int bitDepth) const +double AlfCovariance::calcErrorForCcAlfCoeffs(const int16_t *coeff, const int numCoeff, const int bitDepth) const { double factor = 1 << (bitDepth - 1); double error = 0; @@ -1883,7 +1883,8 @@ void EncAdaptiveLoopFilter::roundFiltCoeff( int *filterCoeffQuant, double *filte } } -void EncAdaptiveLoopFilter::roundFiltCoeffCCALF( int *filterCoeffQuant, double *filterCoeff, const int numCoeff, const int factor ) +void EncAdaptiveLoopFilter::roundFiltCoeffCCALF(int16_t *filterCoeffQuant, double *filterCoeff, const int numCoeff, + const int factor) { for( int i = 0; i < numCoeff; i++ ) { @@ -3308,7 +3309,7 @@ void EncAdaptiveLoopFilter::deriveCcAlfFilterCoeff( ComponentID compID, const Pe using Ty = double[MAX_NUM_ALF_LUMA_COEFF]; double filterCoeffDbl[MAX_NUM_CC_ALF_CHROMA_COEFF]; - int filterCoeffInt[MAX_NUM_CC_ALF_CHROMA_COEFF]; + int16_t filterCoeffInt[MAX_NUM_CC_ALF_CHROMA_COEFF]; std::fill_n(filterCoeffInt, MAX_NUM_CC_ALF_CHROMA_COEFF, 0); @@ -3836,8 +3837,10 @@ void EncAdaptiveLoopFilter::deriveCcAlfFilter( CodingStructure& cs, ComponentID for (int x = 0; x < m_buf->width; x += (1 << log2BlockWidth)) { int ctuIdx = (y >> log2BlockHeight) * m_numCTUsInWidth + (x >> log2BlockWidth); - m_trainingDistortion[filterIdx][ctuIdx] = int(m_ctbDistortionUnfilter[compID][ctuIdx] + - m_alfCovarianceCcAlf[compID - 1][0][0][ctuIdx].calcErrorForCcAlfCoeffs((int*)(ccAlfFilterCoeff[filterIdx]), numCoeff, (m_scaleBits + 1))); + m_trainingDistortion[filterIdx][ctuIdx] = + int(m_ctbDistortionUnfilter[compID][ctuIdx] + + m_alfCovarianceCcAlf[compID - 1][0][0][ctuIdx].calcErrorForCcAlfCoeffs( + ccAlfFilterCoeff[filterIdx], numCoeff, m_scaleBits + 1)); } } #else diff --git a/source/Lib/EncoderLib/EncAdaptiveLoopFilter.h b/source/Lib/EncoderLib/EncAdaptiveLoopFilter.h index cecff4c840677371c42632aa14e26dea53c10947..0aa905998a26b7fd914ff40314fd22b8c69118e4 100644 --- a/source/Lib/EncoderLib/EncAdaptiveLoopFilter.h +++ b/source/Lib/EncoderLib/EncAdaptiveLoopFilter.h @@ -203,7 +203,7 @@ struct AlfCovariance double calculateError( const int *clip, const double *coeff ) const { return calculateError(clip, coeff, numCoeff); } double calculateError( const int *clip, const double *coeff, const int numCoeff ) const; double calcErrorForCoeffs( const int *clip, const int *coeff, const int numCoeff, const int bitDepth ) const; - double calcErrorForCcAlfCoeffs(const int* coeff, const int numCoeff, const int bitDepth) const; + double calcErrorForCcAlfCoeffs(const int16_t *coeff, const int numCoeff, const int bitDepth) const; void getClipMax(const AlfFilterShape& alfShape, int *clip_max) const; void reduceClipCost(const AlfFilterShape& alfShape, int *clip) const; @@ -338,7 +338,7 @@ private: #endif const int numClasses, const int numCoeff, double& distUnfilter ); void roundFiltCoeff( int *filterCoeffQuant, double *filterCoeff, const int numCoeff, const int factor ); - void roundFiltCoeffCCALF( int *filterCoeffQuant, double *filterCoeff, const int numCoeff, const int factor ); + void roundFiltCoeffCCALF(int16_t *filterCoeffQuant, double *filterCoeff, const int numCoeff, const int factor); double getDistCoeffForce0( bool* codedVarBins, double errorForce0CoeffTab[MAX_NUM_ALF_CLASSES][2], int* bitsVarBin, int zeroBitsVarBin, const int numFilters); int lengthUvlc( int uiCode );