From c6092fc7e96e529e92a2ac4ca58b2ca8e8e3ee8b Mon Sep 17 00:00:00 2001
From: Kenneth Andersson <kenneth.r.andersson@ericsson.com>
Date: Fri, 1 Feb 2019 18:02:10 +0100
Subject: [PATCH]  FIX_DB_MAX_TRANSFORM_SIZE

---
 source/Lib/CommonLib/LoopFilter.cpp | 48 +++++++++++++++++++++++++++++
 source/Lib/CommonLib/TypeDef.h      |  1 +
 2 files changed, 49 insertions(+)

diff --git a/source/Lib/CommonLib/LoopFilter.cpp b/source/Lib/CommonLib/LoopFilter.cpp
index 101db8e28..8c7751743 100644
--- a/source/Lib/CommonLib/LoopFilter.cpp
+++ b/source/Lib/CommonLib/LoopFilter.cpp
@@ -353,19 +353,50 @@ void LoopFilter::xDeblockCU( CodingUnit& cu, const DeblockEdgeDir edgeDir )
 
   unsigned int orthogonalLength = 1;
   unsigned int orthogonalIncrement = 1;
+#if FIX_DB_MAX_TRANSFORM_SIZE
+  int maxTsize = 64;
+  maxTsize = 1 << cu.slice->getSPS()->getQuadtreeTULog2MaxSize();
+#endif
 #if JVET_M0471_LONG_DEBLOCKING_FILTERS
+#if FIX_DB_MAX_TRANSFORM_SIZE
+  int maxFilterLengthQ = 7;
+  int maxFilterLengthP = 7;
+  if (implicitTU && maxTsize < 32)
+  {
+    maxFilterLengthQ = 3;
+    maxFilterLengthP = 3;
+  }
+#else
   int maxFilterLength = 7;
+#endif
 #endif
   if (cu.blocks[COMPONENT_Y].valid())
   {
     if (mvSubBlocks)
     {
 #if JVET_M0471_LONG_DEBLOCKING_FILTERS
+#if FIX_DB_MAX_TRANSFORM_SIZE
+      maxFilterLengthQ = std::min(maxFilterLengthQ, 5);
+#else
       maxFilterLength = 5;
+#endif
 #endif
       orthogonalIncrement = subBlockSize / 4;
       orthogonalLength = (edgeDir == EDGE_HOR) ? cu.blocks[COMPONENT_Y].height / 4 : cu.blocks[COMPONENT_Y].width / 4;
     }
+#if FIX_DB_MAX_TRANSFORM_SIZE
+    if ((cu.blocks[COMPONENT_Y].height > maxTsize) && (edgeDir == EDGE_HOR) && !mvSubBlocks)
+    {
+      orthogonalIncrement = maxTsize / 4;
+      orthogonalLength = cu.blocks[COMPONENT_Y].height / 4;
+    }
+    if ((cu.blocks[COMPONENT_Y].width > maxTsize) && (edgeDir == EDGE_VER) && !mvSubBlocks)
+    {
+      orthogonalIncrement = maxTsize / 4;
+      orthogonalLength = cu.blocks[COMPONENT_Y].width / 4;
+
+    }
+#else
     if ((cu.blocks[COMPONENT_Y].height > 64) && (edgeDir == EDGE_HOR) && !mvSubBlocks)
     {
       orthogonalIncrement = 64 / 4;
@@ -377,6 +408,7 @@ void LoopFilter::xDeblockCU( CodingUnit& cu, const DeblockEdgeDir edgeDir )
       orthogonalLength = cu.blocks[COMPONENT_Y].width / 4;
 
     }
+#endif
   }
 
   for (int edge = 0; edge < orthogonalLength; edge += orthogonalIncrement)
@@ -386,15 +418,31 @@ void LoopFilter::xDeblockCU( CodingUnit& cu, const DeblockEdgeDir edgeDir )
 #if JVET_M0471_LONG_DEBLOCKING_FILTERS
       if (edge == 0)
       {
+#if FIX_DB_MAX_TRANSFORM_SIZE 
+        xEdgeFilterLuma(cu, edgeDir, edge, maxFilterLengthP, maxFilterLengthQ);
+#else
         xEdgeFilterLuma(cu, edgeDir, edge, 7, maxFilterLength);
+#endif
       }
       else
       {
+#if FIX_DB_MAX_TRANSFORM_SIZE 
+        if (implicitTU && ((edge % (maxTsize / 4)) == 0))
+#else
         if ( implicitTU && (edge == (64 / 4)) )
+#endif
         {
+#if FIX_DB_MAX_TRANSFORM_SIZE 
+          xEdgeFilterLuma(cu, edgeDir, edge, maxFilterLengthQ, maxFilterLengthQ);
+#else
           xEdgeFilterLuma(cu, edgeDir, edge, maxFilterLength, maxFilterLength);
+#endif
         }
+#if FIX_DB_MAX_TRANSFORM_SIZE 
+        else if ((edge == 2 || edge == (orthogonalLength - 2)) || (implicitTU && (((edge - 2) % ((maxTsize) / 4) == 0) || ((edge + 2) % ((maxTsize) / 4) == 0))))
+#else
         else if ( (edge == 2 || edge == (orthogonalLength - 2)) || (implicitTU && (edge == (56 / 4) || edge == (72 / 4))) )
+#endif
         {
           xEdgeFilterLuma(cu, edgeDir, edge, 2, 2);
         }
diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h
index 507136cdb..5b2baf8cf 100644
--- a/source/Lib/CommonLib/TypeDef.h
+++ b/source/Lib/CommonLib/TypeDef.h
@@ -52,6 +52,7 @@
 
 #define JVET_M0303_IMPLICIT_MTS                           1 // Implicit transform selection (can be enabled with MTSImplicit encoder config parameter)
 
+#define FIX_DB_MAX_TRANSFORM_SIZE                         1
 #define JVET_M0908_CIIP_DB                                1
 #define JVET_M0471_LONG_DEBLOCKING_FILTERS                1 
 #define JVET_M0427_INLOOP_RESHAPER                        1
-- 
GitLab