JVET-O0432: LMCS encoder improvement

cfg/encoder_intra_vtm.cfg

@@ -110,7 +110,9 @@ ALF                          : 1
 IBC                          : 0      # turned off in CTC 
 AllowDisFracMMVD             : 1
 AffineAmvr                   : 0
-LumaReshapeEnable            : 1      # luma reshaping. 0: disable 1:enable 
+LMCSEnable                   : 1      # LMCS: 0: disable, 1:enable
+LMCSSignalType               : 0      # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG
+LMCSUpdateCtrl               : 1      # LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP
 MIP                          : 1
 
 # Fast tools

cfg/encoder_lowdelay_P_vtm.cfg

@@ -126,7 +126,9 @@ MHIntra                      : 1
 IBC                          : 0      # turned off in CTC
 AllowDisFracMMVD             : 1
 AffineAmvr                   : 0
-LumaReshapeEnable            : 1      # luma reshaping. 0: disable 1:enable 
+LMCSEnable                   : 1      # LMCS: 0: disable, 1:enable
+LMCSSignalType               : 0      # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG
+LMCSUpdateCtrl               : 2      # LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP
 MIP                          : 1
 
 # Fast tools

cfg/encoder_lowdelay_vtm.cfg

@@ -130,7 +130,9 @@ Triangle                     : 1
 IBC                          : 0      # turned off in CTC
 AllowDisFracMMVD             : 1
 AffineAmvr                   : 0
-LumaReshapeEnable            : 1      # luma reshaping. 0: disable 1:enable 
+LMCSEnable                   : 1      # LMCS: 0: disable, 1:enable
+LMCSSignalType               : 0      # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG
+LMCSUpdateCtrl               : 2      # LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP
 MIP                          : 1
 
 # Fast tools

cfg/encoder_randomaccess_vtm.cfg

@@ -146,7 +146,9 @@ Triangle                     : 1
 IBC                          : 0      # turned off in CTC
 AllowDisFracMMVD             : 1
 AffineAmvr                   : 1
-LumaReshapeEnable            : 1      # luma reshaping. 0: disable 1:enable 
+LMCSEnable                   : 1      # LMCS: 0: disable, 1:enable
+LMCSSignalType               : 0      # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG
+LMCSUpdateCtrl               : 0      # LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP
 MIP                          : 1
 DMVR                         : 1
 SMVD                         : 1

cfg/per-class/classH1.cfg

@@ -3,7 +3,8 @@ LumaLevelToDeltaQPMode        : 1           # Change luma delta QP based on aver
 isSDR                         : 0           # 1: SDR in PQ container,   0: HDR
 
 # ======= LMCS =======================
-LumaReshapeEnable             : 0            # turned off in HDR CTC 
+LMCSEnable                    : 0           # turned off in HDR CTC 
+LMCSSignalType                : 1           # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG
 
 #======== Chroma QP scale ============
 WCGPPSEnable                  : 1           # enable WCG Chroma scale

cfg/per-class/classH2.cfg

@@ -2,8 +2,8 @@
 LumaLevelToDeltaQPMode        : 0           # Change luma delta QP based on average luma
 
 # ======= LMCS =======================
-LumaReshapeEnable             : 0           # turned off in HDR CTC 
-ReshapeSignalType             : 2           # set signal type to HLG if enabled
+LMCSEnable                    : 1           # turned on in HLG CTC 
+LMCSSignalType                : 2           # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG
 
 #======== Chroma QP scale ============
 WCGPPSEnable                  : 0           # enable WCG Chroma scale

doc/software-manual.tex

@@ -2071,6 +2071,50 @@ Enables or disables the use of low frequency non-separable transform (LFNST).
 Enables or disables the fast encoding of low frequency non-separable transform (LFNST).
 \\
 
+\Option{LMCSEnable} &
+%\ShortOption{\None} &
+\Default{true} &
+Enables or disables the use of LMCS (luma mapping with chroma scaling).
+\\
+
+\Option{LMCSSignalType} &
+%\ShortOption{\None} &
+\Default{0} &
+LMCS signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG.
+\\
+
+\Option{LMCSUpdateCtrl} &
+%\ShortOption{\None} &
+\Default{0} &
+LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP.
+\par
+\begin{tabular}{cp{0.45\textwidth}}
+	0 & Random access: derive a new LMCS model at each IRAP.\\
+	1 & All intra: derive a new LMCS model at each intra slice.\\
+	2 & Low delay: derive a new LMCS model every second. \\
+\end{tabular}
+\\
+
+\Option{LMCSAdpOption} &
+%\ShortOption{\None} &
+\Default{0} &
+Adaptive LMCS mapping derivation options: Options 1 to 4 are for experimental testing purposes and need to set parameter LMCSInitialCW.
+\par
+\begin{tabular}{cp{0.45\textwidth}}
+	0 & Automatic adaptive algorithm (default).\\
+	1 & Derives LMCS mapping with input LMCSInitialCW and enables LMCS for all slices. Uses a static LMCS mapping for low QP ($QP<=22$). \\
+	2 & Derives LMCS mapping with input LMCSInitialCW and enables LMCS only for slices in lowest temporal layer. \\
+	3 & In addition to 1, disables LMCS for intra slices. \\
+	4 & Derives LMCS mapping with input LMCSInitialCW and enables LMCS only for inter slices. \\
+\end{tabular}
+\\
+
+\Option{LMCSInitialCW} &
+%\ShortOption{\None} &
+\Default{0} &
+LMCS initial total codeword (valid values [$0 - 1023$]) to be used in LMCS mapping derivation when LMCSAdpOption is not equal to 0.
+\\
+
 \end{OptionTableNoShorthand}
 
 %%

source/App/EncoderApp/EncAppCfg.cpp

@@ -919,8 +919,18 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] )
   ("VirtualBoundariesPosX",                           cfg_virtualBoundariesPosX,    cfg_virtualBoundariesPosX, "Locations of the vertical virtual boundaries in units of luma samples")
   ("VirtualBoundariesPosY",                           cfg_virtualBoundariesPosY,    cfg_virtualBoundariesPosY, "Locations of the horizontal virtual boundaries in units of luma samples")
   ("EncDbOpt",                                        m_encDbOpt,                                       false, "Encoder optimization with deblocking filter")
+#if JVET_O0432_LMCS_ENCODER
+  ("LMCSEnable",                                      m_lumaReshapeEnable,                              false, "Enable LMCS (luma mapping with chroma scaling")
+  ("LMCSSignalType",                                  m_reshapeSignalType,                                 0u, "Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG")
+  ("LMCSUpdateCtrl",                                  m_updateCtrl,                                         0, "LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP")
+  ("LMCSAdpOption",                                   m_adpOption,                                          0, "LMCS adaptation options: 0:automatic(default),"
+                                                                                                               "1: rsp both (CW66 for QP<=22), 2: rsp TID0 (for all QP),"
+                                                                                                               "3: rsp inter(CW66 for QP<=22), 4: rsp inter(for all QP).")
+  ("LMCSInitialCW",                                   m_initialCW,                                         0u, "LMCS initial total codeword (0~1023) when LMCSAdpOption > 0")
+#else
   ("LumaReshapeEnable",                               m_lumaReshapeEnable,                              false, "Enable Reshaping for Luma Channel")
   ("ReshapeSignalType",                               m_reshapeSignalType,                                 0u, "Input signal type: 0: SDR, 1:PQ, 2:HLG")
+#endif
   ("IntraCMD",                                        m_intraCMD,                                          0u, "IntraChroma MD: 0: none, 1:fixed to default wPSNR weight")
   ("LCTUFast",                                        m_useFastLCTU,                                    false, "Fast methods for large CTU")
   ("FastMrg",                                         m_useFastMrg,                                     false, "Fast methods for inter merge")
@@ -2045,10 +2055,17 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] )
 
   m_reshapeCW.binCW.resize(3);
   m_reshapeCW.rspFps = m_iFrameRate;
+#if !JVET_O0432_LMCS_ENCODER
   m_reshapeCW.rspIntraPeriod = m_iIntraPeriod;
+#endif
   m_reshapeCW.rspPicSize = m_iSourceWidth*m_iSourceHeight;
   m_reshapeCW.rspFpsToIp = std::max(16, 16 * (int)(round((double)m_iFrameRate /16.0)));
   m_reshapeCW.rspBaseQP = m_iQP;
+#if JVET_O0432_LMCS_ENCODER
+  m_reshapeCW.updateCtrl = m_updateCtrl;
+  m_reshapeCW.adpOption = m_adpOption;
+  m_reshapeCW.initialCW = m_initialCW;
+#endif
 #if ENABLE_TRACING
   g_trace_ctx = tracing_init(sTracingFile, sTracingRule);
   if( bTracingChannelsList && g_trace_ctx )
@@ -2434,7 +2451,11 @@ bool EncAppCfg::xCheckParameter()
   {
     m_intraCMD = 1;
   }
+#if JVET_O0432_LMCS_ENCODER
+  else if (m_lumaReshapeEnable && (m_reshapeSignalType == RESHAPE_SIGNAL_SDR || m_reshapeSignalType == RESHAPE_SIGNAL_HLG))
+#else
   else if (m_lumaReshapeEnable && m_reshapeSignalType == RESHAPE_SIGNAL_SDR)
+#endif
   {
     m_intraCMD = 0;
   }
@@ -2442,6 +2463,18 @@ bool EncAppCfg::xCheckParameter()
   {
     m_lumaReshapeEnable = false;
   }
+#if JVET_O0432_LMCS_ENCODER
+  if (m_lumaReshapeEnable)
+  {
+    xConfirmPara(m_updateCtrl < 0, "Min. LMCS Update Control is 0");
+    xConfirmPara(m_updateCtrl > 2, "Max. LMCS Update Control is 2");
+    xConfirmPara(m_adpOption < 0, "Min. LMCS Adaptation Option is 0");
+    xConfirmPara(m_adpOption > 4, "Max. LMCS Adaptation Option is 4");
+    xConfirmPara(m_initialCW < 0, "Min. Initial Total Codeword is 0");
+    xConfirmPara(m_initialCW > 1023, "Max. Initial Total Codeword is 1023");
+    if (m_updateCtrl > 0 && m_adpOption > 2) { m_adpOption -= 2; }
+  }
+#endif
 
   xConfirmPara( m_cbQpOffset < -12,   "Min. Chroma Cb QP Offset is -12" );
   xConfirmPara( m_cbQpOffset >  12,   "Max. Chroma Cb QP Offset is  12" );
@@ -3360,7 +3393,13 @@ void EncAppCfg::xPrintParameter()
     msg(VERBOSE, "Reshape:%d ", m_lumaReshapeEnable);
     if (m_lumaReshapeEnable)
     {
+#if JVET_O0432_LMCS_ENCODER
+      msg(VERBOSE, "(Signal:%s ", m_reshapeSignalType == 0 ? "SDR" : (m_reshapeSignalType == 2 ? "HDR-HLG" : "HDR-PQ"));
+      msg(VERBOSE, "Opt:%d", m_adpOption);
+      if (m_adpOption > 0) { msg(VERBOSE, " CW:%d", m_initialCW); }
+#else
       msg(VERBOSE, "(Sigal:%s ", m_reshapeSignalType==0? "SDR" : "HDR-PQ");
+#endif
       msg(VERBOSE, ") ");
     }
     msg(VERBOSE, "MIP:%d ", m_MIP);

source/App/EncoderApp/EncAppCfg.h

@@ -304,6 +304,11 @@ protected:
   uint32_t  m_reshapeSignalType;
   uint32_t  m_intraCMD;
   ReshapeCW m_reshapeCW;
+#if JVET_O0432_LMCS_ENCODER
+  int       m_updateCtrl;
+  int       m_adpOption;
+  uint32_t  m_initialCW;
+#endif
   bool      m_encDbOpt;
   unsigned  m_uiMaxCUWidth;                                   ///< max. CU width in pixel
   unsigned  m_uiMaxCUHeight;                                  ///< max. CU height in pixel

source/Lib/CommonLib/RdCost.cpp

@@ -2938,7 +2938,11 @@ void RdCost::restoreReshapeLumaLevelToWeightTable()
 
 void RdCost::updateReshapeLumaLevelToWeightTable(SliceReshapeInfo &sliceReshape, Pel *wtTable, double cwt)
 {
+#if JVET_O0432_LMCS_ENCODER
+  if (m_signalType == RESHAPE_SIGNAL_SDR || m_signalType == RESHAPE_SIGNAL_HLG)
+#else
   if (m_signalType == RESHAPE_SIGNAL_SDR)
+#endif
   {
     if (sliceReshape.getSliceReshapeModelPresentFlag())
     {
@@ -2991,7 +2995,11 @@ Distortion RdCost::getWeightedMSE(int compIdx, const Pel org, const Pel cur, con
   }
   // use luma to get weight
   double weight = 1.0;
+#if JVET_O0432_LMCS_ENCODER
+  if (m_signalType == RESHAPE_SIGNAL_SDR || m_signalType == RESHAPE_SIGNAL_HLG)
+#else
   if (m_signalType == RESHAPE_SIGNAL_SDR)
+#endif
   {
     if (compIdx == COMPONENT_Y)
     {

source/Lib/CommonLib/Slice.h

@@ -393,8 +393,15 @@ public:
 struct ReshapeCW
 {
   std::vector<uint32_t> binCW;
+#if JVET_O0432_LMCS_ENCODER
+  int       updateCtrl;
+  int       adpOption;
+  uint32_t  initialCW;
+#endif
   int rspPicSize;
+#if !JVET_O0432_LMCS_ENCODER
   int rspIntraPeriod;
+#endif
   int rspFps;
   int rspBaseQP;
   int rspTid;

source/Lib/CommonLib/TypeDef.h

@@ -116,6 +116,8 @@
 
 #define JVET_O0590_REDUCE_DMVR_ORIG_MV_COST               1 // Reduce the DMVR cost of the original MV
 
+#define JVET_O0432_LMCS_ENCODER                           1 // JVET-O0432: LMCS encoder improvement
+
 #define JVET_O0429_CRS_LAMBDA_FIX                         1 // JVET-O0429: fix encoder lambda rounding used in CRS
 
 #define JVET_O0428_LMCS_CLEANUP                           1 // JVET-O0428: LMCS cleanups

source/Lib/EncoderLib/EncGOP.cpp

@@ -2104,10 +2104,17 @@ void EncGOP::compressGOP( int iPOCLast, int iNumPicRcvd, PicList& rcListPic,
       {
         m_pcReshaper->preAnalyzerHDR(pcPic, pcSlice->getSliceType(), m_pcCfg->getReshapeCW(), m_pcCfg->getDualITree());
       }
+#if JVET_O0432_LMCS_ENCODER
+      else if (m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_SDR || m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_HLG)
+      {
+        m_pcReshaper->preAnalyzerLMCS(pcPic, m_pcCfg->getReshapeSignalType(), pcSlice->getSliceType(), m_pcCfg->getReshapeCW());
+      }
+#else
       else if (m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_SDR)
       {
         m_pcReshaper->preAnalyzerSDR(pcPic, pcSlice->getSliceType(), m_pcCfg->getReshapeCW(), m_pcCfg->getDualITree());
       }
+#endif
       else
       {
         THROW("Reshaper for other signal currently not defined!");
@@ -2120,6 +2127,16 @@ void EncGOP::compressGOP( int iPOCLast, int iNumPicRcvd, PicList& rcListPic,
           m_pcReshaper->initLUTfromdQPModel();
           m_pcEncLib->getRdCost()->updateReshapeLumaLevelToWeightTableChromaMD(m_pcReshaper->getInvLUT());
         }
+#if JVET_O0432_LMCS_ENCODER
+        else if (m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_SDR || m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_HLG)
+        {
+          if (m_pcReshaper->getReshapeFlag())
+          {
+            m_pcReshaper->constructReshaperLMCS();
+            m_pcEncLib->getRdCost()->updateReshapeLumaLevelToWeightTable(m_pcReshaper->getSliceReshaperInfo(), m_pcReshaper->getWeightTable(), m_pcReshaper->getCWeight());
+          }
+        }
+#else
         else if (m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_SDR)
         {
           if (m_pcReshaper->getReshapeFlag())
@@ -2128,6 +2145,7 @@ void EncGOP::compressGOP( int iPOCLast, int iNumPicRcvd, PicList& rcListPic,
             m_pcEncLib->getRdCost()->updateReshapeLumaLevelToWeightTable(m_pcReshaper->getSliceReshaperInfo(), m_pcReshaper->getWeightTable(), m_pcReshaper->getCWeight());
           }
         }
+#endif
         else
         {
           THROW("Reshaper for other signal currently not defined!");
@@ -2158,6 +2176,18 @@ void EncGOP::compressGOP( int iPOCLast, int iNumPicRcvd, PicList& rcListPic,
         {
           m_pcEncLib->getRdCost()->restoreReshapeLumaLevelToWeightTable();
         }
+#if JVET_O0432_LMCS_ENCODER
+        else if (m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_SDR || m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_HLG)
+        {
+          int modIP = pcPic->getPOC() - pcPic->getPOC() / m_pcCfg->getReshapeCW().rspFpsToIp * m_pcCfg->getReshapeCW().rspFpsToIp;
+          if (m_pcReshaper->getReshapeFlag() && m_pcCfg->getReshapeCW().updateCtrl == 2 && modIP == 0)
+          {
+            m_pcReshaper->getSliceReshaperInfo().setSliceReshapeModelPresentFlag(true);
+            m_pcReshaper->constructReshaperLMCS();
+            m_pcEncLib->getRdCost()->updateReshapeLumaLevelToWeightTable(m_pcReshaper->getSliceReshaperInfo(), m_pcReshaper->getWeightTable(), m_pcReshaper->getCWeight());
+          }
+        }
+#else
         else if (m_pcCfg->getReshapeSignalType() == RESHAPE_SIGNAL_SDR)
         {
           int modIP = pcPic->getPOC() - pcPic->getPOC() / m_pcCfg->getReshapeCW().rspFpsToIp * m_pcCfg->getReshapeCW().rspFpsToIp;
@@ -2168,6 +2198,7 @@ void EncGOP::compressGOP( int iPOCLast, int iNumPicRcvd, PicList& rcListPic,
             m_pcEncLib->getRdCost()->updateReshapeLumaLevelToWeightTable(m_pcReshaper->getSliceReshaperInfo(), m_pcReshaper->getWeightTable(), m_pcReshaper->getCWeight());
           }
         }
+#endif
         else
         {
           THROW("Reshaper for other signal currently not defined!");

source/Lib/EncoderLib/EncReshape.h

@@ -49,6 +49,22 @@
 // ====================================================================================================================
 // Class definition
 // ====================================================================================================================
+#if JVET_O0432_LMCS_ENCODER
+struct SeqInfo
+{
+  double binVar[PIC_ANALYZE_CW_BINS];
+  double binHist[PIC_ANALYZE_CW_BINS];
+  double normVar[PIC_ANALYZE_CW_BINS];
+  int    nonZeroCnt;
+  double weightVar;
+  double weightNorm;
+  double minBinVar;
+  double maxBinVar;
+  double meanBinVar;
+  double ratioStdU;
+  double ratioStdV;
+};
+#endif
 
 class EncReshape : public Reshape
 {
@@ -71,6 +87,11 @@ private:
   Pel                     m_cwLumaWeight[PIC_CODE_CW_BINS];
   double                  m_chromaWeight;
   int                     m_chromaAdj;
+#if JVET_O0432_LMCS_ENCODER
+  int                     m_binNum;
+  SeqInfo                 m_srcSeqStats;
+  SeqInfo                 m_rspSeqStats;
+#endif
 public:
 
   EncReshape();
@@ -81,15 +102,31 @@ public:
 
   bool getSrcReshaped() { return m_srcReshaped; }
   void setSrcReshaped(bool b) { m_srcReshaped = b; }
+#if JVET_O0432_LMCS_ENCODER
+  void initSeqStats(SeqInfo &stats);
+  void calcSeqStats(Picture *pcPic, SeqInfo &stats);
+  void preAnalyzerLMCS(Picture *pcPic, const uint32_t signalType, const SliceType sliceType, const ReshapeCW& reshapeCW);
+#else
   void preAnalyzerSDR(Picture *pcPic, const SliceType sliceType, const ReshapeCW& reshapeCW, bool isDualT);
+#endif
   void preAnalyzerHDR(Picture *pcPic, const SliceType sliceType, const ReshapeCW& reshapeCW, bool isDualT);
   void bubbleSortDsd(double *array, int * idx, int n);
   void swap(int *xp, int *yp) { int temp = *xp;  *xp = *yp;  *yp = temp; }
   void swap(double *xp, double *yp) { double temp = *xp;  *xp = *yp;  *yp = temp; }
+#if JVET_O0432_LMCS_ENCODER
+  void cwPerturbation(int startBinIdx, int endBinIdx, uint16_t maxCW);
+  void cwReduction(int startBinIdx, int endBinIdx);
+  void deriveReshapeParametersSDR(bool *intraAdp, bool *interAdp);
+#else
   void deriveReshapeParametersSDRfromStats(uint32_t *, double*, double* reshapeTH1, double* reshapeTH2, bool *intraAdp, bool *interAdp);
+#endif
   void deriveReshapeParameters(double *array, int start, int end, ReshapeCW respCW, double &alpha, double &beta);
   void initLUTfromdQPModel();
+#if JVET_O0432_LMCS_ENCODER
+  void constructReshaperLMCS();
+#else
   void constructReshaperSDR();
+#endif
   ReshapeCW * getReshapeCW() { return &m_reshapeCW; }
   Pel * getWeightTable() { return m_cwLumaWeight; }
   double getCWeight() { return m_chromaWeight; }