LoopFilter.cpp 63.2 KB
Newer Older
1
2
3
4
5
/* The copyright in this software is being made available under the BSD
 * License, included below. This software may be subject to other third party
 * and contributor rights, including patent rights, and no such rights are
 * granted under this license.
 *
6
 * Copyright (c) 2010-2019, ITU/ISO/IEC
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *  * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
 *    be used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

/** \file     LoopFilter.cpp
    \brief    deblocking filter
*/

#include "LoopFilter.h"
#include "Slice.h"
#include "Mv.h"
#include "Unit.h"
#include "UnitTools.h"
#include "UnitPartitioner.h"
#include "dtrace_codingstruct.h"
#include "dtrace_buffer.h"

//! \ingroup CommonLib
//! \{

// ====================================================================================================================
// Constants
// ====================================================================================================================

//#define   EDGE_VER    0
//#define   EDGE_HOR    1

#define DEBLOCK_SMALLEST_BLOCK  8


#define DEFAULT_INTRA_TC_OFFSET 2 ///< Default intra TC offset

// ====================================================================================================================
// Tables
// ====================================================================================================================

const uint8_t LoopFilter::sm_tcTable[MAX_QP + 1 + DEFAULT_INTRA_TC_OFFSET] =
{
Xiang Li's avatar
Xiang Li committed
68
69
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,5,5,6,6,7,8,9,10,11,13,14,16,18,20,22,25
  , 28, 31, 35, 39, 44, 50, 56, 63, 70, 79, 88, 99
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
};

const uint8_t LoopFilter::sm_betaTable[MAX_QP + 1] =
{
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64
  , 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88
};

inline static uint32_t getRasterIdx(const Position& pos, const PreCalcValues& pcv)
{
  return ( ( pos.x & pcv.maxCUWidthMask ) >> pcv.minCUWidthLog2 ) + ( ( pos.y & pcv.maxCUHeightMask ) >> pcv.minCUHeightLog2 ) * pcv.partsInCtuWidth;
}

// ====================================================================================================================
// utility functions
// ====================================================================================================================

static bool isAvailableLeft( const CodingUnit& cu, const CodingUnit& cu2, const bool bEnforceSliceRestriction, const bool bEnforceTileRestriction )
{
  return ( ( !bEnforceSliceRestriction || CU::isSameSlice( cu, cu2 ) ) && ( !bEnforceTileRestriction || CU::isSameTile( cu, cu2 ) ) );
}

static bool isAvailableAbove( const CodingUnit& cu, const CodingUnit& cu2, const bool bEnforceSliceRestriction, const bool bEnforceTileRestriction )
{
  return ( !bEnforceSliceRestriction || CU::isSameSlice( cu, cu2 ) ) && ( !bEnforceTileRestriction || CU::isSameTile( cu, cu2 ) );
}


// ====================================================================================================================
// Constructor / destructor / create / destroy
// ====================================================================================================================

LoopFilter::LoopFilter()
{
}

LoopFilter::~LoopFilter()
{
}

// ====================================================================================================================
// Public member functions
// ====================================================================================================================
void LoopFilter::create( const unsigned uiMaxCUDepth )
{
  destroy();
  const unsigned numPartitions = 1 << ( uiMaxCUDepth << 1 );
  for( int edgeDir = 0; edgeDir < NUM_EDGE_DIR; edgeDir++ )
  {
    m_aapucBS       [edgeDir].resize( numPartitions );
    m_aapbEdgeFilter[edgeDir].resize( numPartitions );
  }
Nan Hu's avatar
Nan Hu committed
122
  m_enc = false;
Nan Hu's avatar
M0428    
Nan Hu committed
123
124
125
126
127
128
129
}

void LoopFilter::initEncPicYuvBuffer(ChromaFormat chromaFormat, int lumaWidth, int lumaHeight)
{
  const UnitArea picArea(chromaFormat, Area(0, 0, lumaWidth, lumaHeight));
  m_encPicYuvBuffer.destroy();
  m_encPicYuvBuffer.create(picArea);
130
131
132
133
134
135
136
137
138
}

void LoopFilter::destroy()
{
  for( int edgeDir = 0; edgeDir < NUM_EDGE_DIR; edgeDir++ )
  {
    m_aapucBS       [edgeDir].clear();
    m_aapbEdgeFilter[edgeDir].clear();
  }
Nan Hu's avatar
M0428    
Nan Hu committed
139
  m_encPicYuvBuffer.destroy();
140
141
142
143
144
145
146
147
148
149
150
}

/**
 - call deblocking function for every CU
 .
 \param  pcPic   picture class (Pic) pointer
 */
void LoopFilter::loopFilterPic( CodingStructure& cs
                                )
{
  const PreCalcValues& pcv = *cs.pcv;
151
152
  m_shiftHor = ::getComponentScaleX( COMPONENT_Cb, cs.pcv->chrFormat );
  m_shiftVer = ::getComponentScaleY( COMPONENT_Cb, cs.pcv->chrFormat );
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172

  DTRACE_UPDATE( g_trace_ctx, ( std::make_pair( "poc", cs.slice->getPOC() ) ) );
#if ENABLE_TRACING
  for( int y = 0; y < pcv.heightInCtus; y++ )
  {
    for( int x = 0; x < pcv.widthInCtus; x++ )
    {
      const UnitArea ctuArea( pcv.chrFormat, Area( x << pcv.maxCUWidthLog2, y << pcv.maxCUHeightLog2, pcv.maxCUWidth, pcv.maxCUWidth ) );
      DTRACE    ( g_trace_ctx, D_CRC, "CTU %d %d", ctuArea.Y().x, ctuArea.Y().y );
      DTRACE_CRC( g_trace_ctx, D_CRC, cs, cs.picture->getRecoBuf( clipArea( ctuArea, *cs.picture ) ), &ctuArea.Y() );
    }
  }
#endif

  for( int y = 0; y < pcv.heightInCtus; y++ )
  {
    for( int x = 0; x < pcv.widthInCtus; x++ )
    {
      memset( m_aapucBS       [EDGE_VER].data(), 0,     m_aapucBS       [EDGE_VER].byte_size() );
      memset( m_aapbEdgeFilter[EDGE_VER].data(), false, m_aapbEdgeFilter[EDGE_VER].byte_size() );
173
174
175
176
177
      memset( m_maxFilterLengthP, 0, sizeof(m_maxFilterLengthP) );
      memset( m_maxFilterLengthQ, 0, sizeof(m_maxFilterLengthQ) );
      memset( m_transformEdge, false, sizeof(m_transformEdge) );
      m_ctuXLumaSamples = x << pcv.maxCUWidthLog2;
      m_ctuYLumaSamples = y << pcv.maxCUHeightLog2;
178
179

      const UnitArea ctuArea( pcv.chrFormat, Area( x << pcv.maxCUWidthLog2, y << pcv.maxCUHeightLog2, pcv.maxCUWidth, pcv.maxCUWidth ) );
180
181
      CodingUnit* firstCU = cs.getCU( ctuArea.lumaPos(), CH_L);
      cs.slice = firstCU->slice;
182
183
184
185
186
187
188
189
190
191
192

      // CU-based deblocking
      for( auto &currCU : cs.traverseCUs( CS::getArea( cs, ctuArea, CH_L ), CH_L ) )
      {
        xDeblockCU( currCU, EDGE_VER );
      }

      if( CS::isDualITree( cs ) )
      {
        memset( m_aapucBS       [EDGE_VER].data(), 0,     m_aapucBS       [EDGE_VER].byte_size() );
        memset( m_aapbEdgeFilter[EDGE_VER].data(), false, m_aapbEdgeFilter[EDGE_VER].byte_size() );
193
194
195
        memset( m_maxFilterLengthP, 0, sizeof(m_maxFilterLengthP) );
        memset( m_maxFilterLengthQ, 0, sizeof(m_maxFilterLengthQ) );
        memset( m_transformEdge, false, sizeof(m_transformEdge) );
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211

        for( auto &currCU : cs.traverseCUs( CS::getArea( cs, ctuArea, CH_C ), CH_C ) )
        {
          xDeblockCU( currCU, EDGE_VER );
        }
      }
    }
  }

  // Vertical filtering
  for( int y = 0; y < pcv.heightInCtus; y++ )
  {
    for( int x = 0; x < pcv.widthInCtus; x++ )
    {
      memset( m_aapucBS       [EDGE_HOR].data(), 0,     m_aapucBS       [EDGE_HOR].byte_size() );
      memset( m_aapbEdgeFilter[EDGE_HOR].data(), false, m_aapbEdgeFilter[EDGE_HOR].byte_size() );
212
213
214
215
216
      memset( m_maxFilterLengthP, 0, sizeof(m_maxFilterLengthP) );
      memset( m_maxFilterLengthQ, 0, sizeof(m_maxFilterLengthQ) );
      memset( m_transformEdge, false, sizeof(m_transformEdge) );
      m_ctuXLumaSamples = x << pcv.maxCUWidthLog2;
      m_ctuYLumaSamples = y << pcv.maxCUHeightLog2;
217
218

      const UnitArea ctuArea( pcv.chrFormat, Area( x << pcv.maxCUWidthLog2, y << pcv.maxCUHeightLog2, pcv.maxCUWidth, pcv.maxCUWidth ) );
219
220
      CodingUnit* firstCU = cs.getCU( ctuArea.lumaPos(), CH_L);
      cs.slice = firstCU->slice;
221
222
223
224
225
226
227
228
229
230
231

      // CU-based deblocking
      for( auto &currCU : cs.traverseCUs( CS::getArea( cs, ctuArea, CH_L ), CH_L ) )
      {
        xDeblockCU( currCU, EDGE_HOR );
      }

      if( CS::isDualITree( cs ) )
      {
        memset( m_aapucBS       [EDGE_HOR].data(), 0,     m_aapucBS       [EDGE_HOR].byte_size() );
        memset( m_aapbEdgeFilter[EDGE_HOR].data(), false, m_aapbEdgeFilter[EDGE_HOR].byte_size() );
232
233
234
        memset( m_maxFilterLengthP, 0, sizeof(m_maxFilterLengthP) );
        memset( m_maxFilterLengthQ, 0, sizeof(m_maxFilterLengthQ) );
        memset( m_transformEdge, false, sizeof(m_transformEdge) );
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267

        for( auto &currCU : cs.traverseCUs( CS::getArea( cs, ctuArea, CH_C ), CH_C ) )
        {
          xDeblockCU( currCU, EDGE_HOR );
        }
      }
    }
  }

  DTRACE_PIC_COMP(D_REC_CB_LUMA_LF,   cs, cs.getRecoBuf(), COMPONENT_Y);
  DTRACE_PIC_COMP(D_REC_CB_CHROMA_LF, cs, cs.getRecoBuf(), COMPONENT_Cb);
  DTRACE_PIC_COMP(D_REC_CB_CHROMA_LF, cs, cs.getRecoBuf(), COMPONENT_Cr);

  DTRACE    ( g_trace_ctx, D_CRC, "LoopFilter" );
  DTRACE_CRC( g_trace_ctx, D_CRC, cs, cs.getRecoBuf() );
}


// ====================================================================================================================
// Protected member functions
// ====================================================================================================================

/**
 Deblocking filter process in CU-based (the same function as conventional's)

 \param cu               the CU to be deblocked
 \param edgeDir          the direction of the edge in block boundary (horizontal/vertical), which is added newly
*/
void LoopFilter::xDeblockCU( CodingUnit& cu, const DeblockEdgeDir edgeDir )
{
  const PreCalcValues& pcv = *cu.cs->pcv;
  const Area area          = cu.Y().valid() ? cu.Y() : Area( recalcPosition( cu.chromaFormat, cu.chType, CHANNEL_TYPE_LUMA, cu.blocks[cu.chType].pos() ), recalcSize( cu.chromaFormat, cu.chType, CHANNEL_TYPE_LUMA, cu.blocks[cu.chType].size() ) );

268
269
270
271
272
  bool horEdgeFilter = false, verEdgeFilter = false;
  int  numHorVirBndry = 0, numVerVirBndry = 0;
  int  horVirBndryPos[] = { 0, 0, 0 };
  int  verVirBndryPos[] = { 0, 0, 0 };

Sheng-Yen Lin's avatar
Sheng-Yen Lin committed
273
  bool isCuCrossedByVirtualBoundaries = isCrossedByVirtualBoundaries( area.x, area.y, area.width, area.height, numHorVirBndry, numVerVirBndry, horVirBndryPos, verVirBndryPos, cu.cs->slice->getPPS() );
274

275
  xSetLoopfilterParam( cu );
276
277
  static_vector<int, 2*MAX_CU_SIZE> edgeIdx;
  edgeIdx.clear();
278
279
280
  for( auto &currTU : CU::traverseTUs( cu ) )
  {
    const Area& areaTu    = cu.Y().valid() ? currTU.block( COMPONENT_Y ) : area;
281
282
    verEdgeFilter = m_stLFCUParam.internalEdge;
    horEdgeFilter = m_stLFCUParam.internalEdge;
Sheng-Yen Lin's avatar
Sheng-Yen Lin committed
283
    if( isCuCrossedByVirtualBoundaries )
284
285
286
287
288
    {
      xDeriveEdgefilterParam( areaTu.x, areaTu.y, numVerVirBndry, numHorVirBndry, verVirBndryPos, horVirBndryPos, verEdgeFilter, horEdgeFilter );
    }
    xSetEdgefilterMultiple( cu, EDGE_VER, areaTu, verEdgeFilter );
    xSetEdgefilterMultiple( cu, EDGE_HOR, areaTu, horEdgeFilter );
289
290
    xSetMaxFilterLengthPQFromTransformSizes( edgeDir, cu, currTU );
    edgeIdx.push_back( ( edgeDir == EDGE_HOR ) ? ( currTU.blocks[cu.chType].y - cu.blocks[cu.chType].y ) / 4 : ( currTU.blocks[cu.chType].x - cu.blocks[cu.chType].x ) / 4 );
291
292
  }

293
294
  bool mvSubBlocks = false;
  int subBlockSize = 8;
295
296
297
298
299
300
  for( auto &currPU : CU::traversePUs( cu ) )
  {
    const Area& areaPu = cu.Y().valid() ? currPU.block( COMPONENT_Y ) : area;
    const bool xOff    = currPU.blocks[cu.chType].x != cu.blocks[cu.chType].x;
    const bool yOff    = currPU.blocks[cu.chType].y != cu.blocks[cu.chType].y;

301
302
    verEdgeFilter = (xOff ? m_stLFCUParam.internalEdge : m_stLFCUParam.leftEdge);
    horEdgeFilter = (yOff ? m_stLFCUParam.internalEdge : m_stLFCUParam.topEdge);
Sheng-Yen Lin's avatar
Sheng-Yen Lin committed
303
    if( isCuCrossedByVirtualBoundaries )
304
305
306
307
308
309
    {
      xDeriveEdgefilterParam( areaPu.x, areaPu.y, numVerVirBndry, numHorVirBndry, verVirBndryPos, horVirBndryPos, verEdgeFilter, horEdgeFilter );
    }

    xSetEdgefilterMultiple( cu, EDGE_VER, areaPu, verEdgeFilter, xOff );
    xSetEdgefilterMultiple( cu, EDGE_HOR, areaPu, horEdgeFilter, yOff );
310
    edgeIdx.push_back( ( edgeDir == EDGE_HOR ) ? ( currPU.blocks[cu.chType].y - cu.blocks[cu.chType].y ) / 4 : ( currPU.blocks[cu.chType].x - cu.blocks[cu.chType].x ) / 4 );
311

312
313
314
315
316
317
318
319
    if ((currPU.mergeFlag && (currPU.mergeType == MRG_TYPE_SUBPU_ATMVP)) || cu.affine)
    {
      mvSubBlocks = true;
      if (edgeDir == EDGE_HOR)
      {
        for (uint32_t off = subBlockSize; off < areaPu.height; off += subBlockSize)
        {
          const Area mvBlockH(cu.Y().x, cu.Y().y + off, cu.Y().width, pcv.minCUHeight);
320
          horEdgeFilter = m_stLFCUParam.internalEdge;
Sheng-Yen Lin's avatar
Sheng-Yen Lin committed
321
          if( isCuCrossedByVirtualBoundaries )
322
323
324
325
326
          {
            xDeriveEdgefilterParam( mvBlockH.x, mvBlockH.y, 0, numHorVirBndry, verVirBndryPos, horVirBndryPos, verEdgeFilter, horEdgeFilter );
          }

          xSetEdgefilterMultiple(cu, EDGE_HOR, mvBlockH, horEdgeFilter, 1);
327
          edgeIdx.push_back( ( currPU.blocks[cu.chType].y + off - cu.blocks[cu.chType].y ) / 4 );
328
329
330
331
332
333
334
        }
      }
      else
      {
        for (uint32_t off = subBlockSize; off < areaPu.width; off += subBlockSize)
        {
          const Area mvBlockV(cu.Y().x + off, cu.Y().y, pcv.minCUWidth, cu.Y().height);
335
          verEdgeFilter = m_stLFCUParam.internalEdge;
Sheng-Yen Lin's avatar
Sheng-Yen Lin committed
336
          if( isCuCrossedByVirtualBoundaries )
337
338
339
340
341
          {
            xDeriveEdgefilterParam( mvBlockV.x, mvBlockV.y, numVerVirBndry, 0, verVirBndryPos, horVirBndryPos, verEdgeFilter, horEdgeFilter );
          }

          xSetEdgefilterMultiple(cu, EDGE_VER, mvBlockV, verEdgeFilter, 1);
342
          edgeIdx.push_back( ( currPU.blocks[cu.chType].x + off - cu.blocks[cu.chType].x ) / 4 );
343
        }
344
345
      }
    }
346

347
    xSetMaxFilterLengthPQForCodingSubBlocks( edgeDir, cu, currPU, mvSubBlocks, subBlockSize, areaPu );
348
  }
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377

  const unsigned uiPelsInPart = pcv.minCUWidth;

  for( int y = 0; y < area.height; y += uiPelsInPart )
  {
    for( int x = 0; x < area.width; x += uiPelsInPart )
    {
      unsigned uiBSCheck = 1;
      const Position localPos  { area.x + x, area.y + y };
      const unsigned rasterIdx = getRasterIdx( localPos, pcv );

      if( m_aapbEdgeFilter[edgeDir][rasterIdx] && uiBSCheck )
      {
        m_aapucBS[edgeDir][rasterIdx] = xGetBoundaryStrengthSingle( cu, edgeDir, localPos );
      }
    }
  }

  if (edgeDir == EDGE_HOR)
  {
    if (!((cu.block(COMPONENT_Y).y % 8) == 0))
      return;
  }
  else
  {
    if (!((cu.block(COMPONENT_Y).x % 8) == 0))
      return;
  }

378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400

  std::sort( edgeIdx.begin(), edgeIdx.end() );
  int prevEdgeIdx = -1;
  for ( const int& edge : edgeIdx )
  {
    if ( edge == prevEdgeIdx ) // skip duplicate edgeIdx marked by both transform and coding subblock processes
    {
      continue;
    }
    prevEdgeIdx = edge;

    if ( cu.blocks[COMPONENT_Y].valid() )
    {
      xEdgeFilterLuma( cu, edgeDir, edge );
    }
    if ( cu.blocks[COMPONENT_Cb].valid() && pcv.chrFormat != CHROMA_400 )
    {
      if ( !cu.ispMode || edge == 0 )
      {
        xEdgeFilterChroma( cu, edgeDir, edge );
      }
    }
  }
401
402
}

403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
inline bool LoopFilter::isCrossedByVirtualBoundaries(const int xPos, const int yPos, const int width, const int height, int& numHorVirBndry, int& numVerVirBndry, int horVirBndryPos[], int verVirBndryPos[], const PPS* pps)
{
  numHorVirBndry = 0; numVerVirBndry = 0;
  if (pps->getLoopFilterAcrossVirtualBoundariesDisabledFlag())
  {
    for (int i = 0; i < pps->getNumHorVirtualBoundaries(); i++)
    {
      if (yPos <= pps->getVirtualBoundariesPosY(i) && pps->getVirtualBoundariesPosY(i) < yPos + height)
      {
        horVirBndryPos[numHorVirBndry++] = pps->getVirtualBoundariesPosY(i);
      }
    }
    for (int i = 0; i < pps->getNumVerVirtualBoundaries(); i++)
    {
      if (xPos <= pps->getVirtualBoundariesPosX(i) && pps->getVirtualBoundariesPosX(i) < xPos + width)
      {
        verVirBndryPos[numVerVirBndry++] = pps->getVirtualBoundariesPosX(i);
      }
    }
  }
  return numHorVirBndry > 0 || numVerVirBndry > 0;
}

inline void LoopFilter::xDeriveEdgefilterParam( const int xPos, const int yPos, const int numVerVirBndry, const int numHorVirBndry, const int verVirBndryPos[], const int horVirBndryPos[], bool &verEdgeFilter, bool &horEdgeFilter )
{
  for (int i = 0; i < numVerVirBndry; i++)
  {
    if (verVirBndryPos[i] == xPos)
    {
      verEdgeFilter = false;
      break;
    }
  }

  for (int i = 0; i < numHorVirBndry; i++)
  {
    if (horVirBndryPos[i] == yPos)
    {
      horEdgeFilter = false;
      break;
    }
  }
}

447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
void LoopFilter::xSetMaxFilterLengthPQFromTransformSizes( const DeblockEdgeDir edgeDir, const CodingUnit& cu, const TransformUnit& currTU )
{
  const TransformUnit& tuQ = currTU;

  if ( edgeDir == EDGE_HOR )
  {
    for ( int cIdx = 0; cIdx < MAX_NUM_COMPONENT; cIdx++ ) // per component
    {
      const ComponentID comp = ComponentID(cIdx);
      const ChannelType ch   = toChannelType(comp);
      const int shiftHor     = ( ( ch == CH_L ) ? 0 : m_shiftHor );
      const int shiftVer     = ( ( ch == CH_L ) ? 0 : m_shiftVer );
      const int ctuXOff      = currTU.block(comp).x - ( m_ctuXLumaSamples >> shiftHor ); // x offset from left edge of CTU in respective channel sample units
      const int ctuYOff      = currTU.block(comp).y - ( m_ctuYLumaSamples >> shiftVer ); // y offset from top edge of CTU in respective channel sample units
      const int minCUWidth   = cu.cs->pcv->minCUWidth >> shiftHor;
      if ( currTU.block(comp).valid() && ( ( currTU.block(comp).y == cu.block(comp).y ) ? m_stLFCUParam.topEdge : m_stLFCUParam.internalEdge ) ) // Edge deblocking needs to be recomputed since ISP contains whole CU chroma transforms in last TU of the CU
      {
        for ( int x = 0; x < currTU.blocks[cIdx].width; x += minCUWidth )
        {
          const Position  posQ     = Position( currTU.blocks[ch].x + x, currTU.blocks[ch].y );
          const Position  posP     = posQ.offset( 0, -1 );
          const int sizeQSide      = tuQ.block(comp).height;
          const TransformUnit& tuP = *cu.cs->getTU( posP, ch );
          const int sizePSide      = tuP.block(comp).height;
          m_transformEdge[cIdx][ctuXOff+x][ctuYOff] = true;

          if ( comp == COMPONENT_Y )
          {
            m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff] = ( sizeQSide >= 32 ) ? 7 : 3;
            m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff] = ( sizePSide >= 32 ) ? 7 : 3;
          }
          else
          {
            m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff] = ( sizeQSide >= 8 && sizePSide >= 8 ) ? 3 : 1;
            m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff] = ( sizeQSide >= 8 && sizePSide >= 8 ) ? 3 : 1;
          }
        }
      }
    }
  }
  if ( edgeDir == EDGE_VER )
  {
    for ( int cIdx = 0; cIdx < MAX_NUM_COMPONENT; cIdx++ ) // per component
    {
      const ComponentID comp = ComponentID(cIdx);
      const ChannelType ch   = toChannelType(comp);
      const int shiftHor     = ( ( ch == CH_L ) ? 0 : m_shiftHor );
      const int shiftVer     = ( ( ch == CH_L ) ? 0 : m_shiftVer );
      const int ctuXOff      = currTU.block(comp).x - ( m_ctuXLumaSamples >> shiftHor ); // x offset from left edge of CTU in respective channel sample units
      const int ctuYOff      = currTU.block(comp).y - ( m_ctuYLumaSamples >> shiftVer ); // y offset from top edge of CTU in respective channel sample units
      const int minCUHeight  = cu.cs->pcv->minCUHeight >> shiftVer;
      if ( currTU.block(comp).valid() && ( ( currTU.block(comp).x == cu.block(comp).x ) ? m_stLFCUParam.leftEdge : m_stLFCUParam.internalEdge ) ) // Edge deblocking needs to be recomputed since ISP contains whole CU chroma transforms in last TU of the CU
      {
        for ( int y = 0; y < currTU.blocks[cIdx].height; y += minCUHeight )
        {
          const Position  posQ     = Position( currTU.blocks[ch].x, currTU.blocks[ch].y + y );
          const Position  posP     = posQ.offset( -1, 0 );
          const int sizeQSide      = tuQ.block(comp).width;
          const TransformUnit& tuP = *cu.cs->getTU( posP, ch );
          const int sizePSide      = tuP.block(comp).width;
          m_transformEdge[cIdx][ctuXOff][ctuYOff+y] = true;

          if ( comp == COMPONENT_Y )
          {
            m_maxFilterLengthQ[cIdx][ctuXOff][ctuYOff+y] = ( sizeQSide >= 32 ) ? 7 : 3;
            m_maxFilterLengthP[cIdx][ctuXOff][ctuYOff+y] = ( sizePSide >= 32 ) ? 7 : 3;
          }
          else
          {
            m_maxFilterLengthQ[cIdx][ctuXOff][ctuYOff+y] = ( sizeQSide >= 8 && sizePSide >= 8 ) ? 3 : 1;
            m_maxFilterLengthP[cIdx][ctuXOff][ctuYOff+y] = ( sizeQSide >= 8 && sizePSide >= 8 ) ? 3 : 1;
          }
        }
      }
    }
  }
}

void LoopFilter::xSetMaxFilterLengthPQForCodingSubBlocks( const DeblockEdgeDir edgeDir, const CodingUnit& cu, const PredictionUnit& currPU, const bool& mvSubBlocks, const int& subBlockSize, const Area& areaPu )
{
  if ( mvSubBlocks && currPU.Y().valid() )
  {
    const int cIdx         = 0;
    const ComponentID comp = ComponentID(cIdx);
    const int ctuYOff      = currPU.block(comp).y - m_ctuYLumaSamples; // y offset from top edge of CTU in luma samples
    const int ctuXOff      = currPU.block(comp).x - m_ctuXLumaSamples; // x offset from left edge of CTU in luma samples
    const int minCUWidth   = cu.cs->pcv->minCUWidth;
    const int minCUHeight  = cu.cs->pcv->minCUHeight;
    if ( edgeDir == EDGE_HOR )
    {
      for ( int y = 0; y < areaPu.height; y += subBlockSize )
      {
        for ( int x = 0; x < areaPu.width; x += minCUWidth )
        {
          if ( m_transformEdge[cIdx][ctuXOff+x][ctuYOff+y] )
          {
            m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff+y] = std::min<int>(m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff+y], 5);
            if ( y > 0 )
            {
              m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff+y] = std::min<int>(m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff+y], 5);
            }
          }
          else if (y > 0 && ( m_transformEdge[cIdx][ctuXOff+x][ctuYOff+y-8] || (( y + 8 ) >= areaPu.height) || m_transformEdge[cIdx][ctuXOff+x][ctuYOff+y+8] )) // adjacent to transform edge on 8x8 grid
          {
            m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff+y] = 2;
            m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff+y] = 2;
          }
          else
          {
            m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff+y] = 3;
            m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff+y] = 3;
          }
        }
      }
    }
    else // edgeDir == EDGE_VER
    {
      for ( int x = 0; x < areaPu.width; x += subBlockSize )
      {
        for ( int y = 0; y < areaPu.height; y += minCUHeight )
        {
          if ( m_transformEdge[cIdx][ctuXOff+x][ctuYOff+y] )
          {
            m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff+y] = std::min<int>(m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff+y], 5);
            if ( x > 0 )
            {
              m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff+y] = std::min<int>(m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff+y], 5);
            }
          }
          else if ( x > 0 && ( m_transformEdge[cIdx][ctuXOff+x-8][ctuYOff+y] || ( (x + 8) >= areaPu.width ) || m_transformEdge[cIdx][ctuXOff+x+8][ctuYOff+y] ) ) // adjacent to transform edge on 8x8 grid
          {
            m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff+y] = 2;
            m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff+y] = 2;
          }
          else
          {
            m_maxFilterLengthQ[cIdx][ctuXOff+x][ctuYOff+y] = 3;
            m_maxFilterLengthP[cIdx][ctuXOff+x][ctuYOff+y] = 3;
          }
        }
      }
    }
  }
}
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627

void LoopFilter::xSetEdgefilterMultiple( const CodingUnit&    cu,
                                         const DeblockEdgeDir edgeDir,
                                         const Area&          area,
                                         const bool           bValue,
                                         const bool           EdgeIdx )
{
  const PreCalcValues& pcv = *cu.cs->pcv;

  const unsigned uiAdd     = ( edgeDir == EDGE_VER ) ? pcv.partsInCtuWidth : 1;
  const unsigned uiNumElem = ( edgeDir == EDGE_VER ) ? ( area.height / pcv.minCUHeight ) : ( area.width / pcv.minCUWidth );
  unsigned uiBsIdx         = getRasterIdx( area, pcv );

  for( int ui = 0; ui < uiNumElem; ui++ )
  {
    m_aapbEdgeFilter[edgeDir][uiBsIdx] = bValue;
    if( ! EdgeIdx )
    {
      m_aapucBS[edgeDir][uiBsIdx] = bValue;
    }
    uiBsIdx += uiAdd;
  }
}
void LoopFilter::xSetLoopfilterParam( const CodingUnit& cu )
{
  const Slice& slice = *cu.slice;
  const PPS&   pps   = *cu.cs->pps;

  if( slice.getDeblockingFilterDisable() )
  {
    m_stLFCUParam.leftEdge = m_stLFCUParam.topEdge = m_stLFCUParam.internalEdge = false;
    return;
  }

  const Position& pos = cu.blocks[cu.chType].pos();

  m_stLFCUParam.internalEdge = true;
Karsten Suehring's avatar
Karsten Suehring committed
628
629
  m_stLFCUParam.leftEdge     = ( 0 < pos.x ) && isAvailableLeft ( cu, *cu.cs->getCU( pos.offset( -1,  0 ), cu.chType ), !slice.getLFCrossSliceBoundaryFlag(), !pps.getLoopFilterAcrossBricksEnabledFlag() );
  m_stLFCUParam.topEdge      = ( 0 < pos.y ) && isAvailableAbove( cu, *cu.cs->getCU( pos.offset(  0, -1 ), cu.chType ), !slice.getLFCrossSliceBoundaryFlag(), !pps.getLoopFilterAcrossBricksEnabledFlag() );
630
631
632
633
}

unsigned LoopFilter::xGetBoundaryStrengthSingle ( const CodingUnit& cu, const DeblockEdgeDir edgeDir, const Position& localPos ) const
{
634
635
  // The boundary strength that is output by the function xGetBoundaryStrengthSingle is a multi component boundary strength that contains boundary strength for luma (bits 0 to 1), cb (bits 2 to 3) and cr (bits 4 to 5).

636
637
  const Slice& sliceQ = *cu.slice;

Xiaozhong Xu's avatar
Xiaozhong Xu committed
638
639
640
  int shiftHor = cu.Y().valid() ? 0 : ::getComponentScaleX(COMPONENT_Cb, cu.firstPU->chromaFormat);
  int shiftVer = cu.Y().valid() ? 0 : ::getComponentScaleY(COMPONENT_Cb, cu.firstPU->chromaFormat);
  const Position& posQ = Position{ localPos.x >> shiftHor,  localPos.y >> shiftVer };
641
642
643
  const Position  posP  = ( edgeDir == EDGE_VER ) ? posQ.offset( -1, 0 ) : posQ.offset( 0, -1 );

  const CodingUnit& cuQ = cu;
644
  const CodingUnit& cuP = *cu.cs->getCU( posP, cu.chType );
645

646
647
648
649
650
  if( ( MODE_INTRA == cuP.predMode && cuP.bdpcmMode ) && ( MODE_INTRA == cuQ.predMode && cuQ.bdpcmMode ) )
  {
    return 0;
  }

651
652
653
  //-- Set BS for Intra MB : BS = 4 or 3
  if( ( MODE_INTRA == cuP.predMode ) || ( MODE_INTRA == cuQ.predMode ) )
  {
654
    return (BsSet(2, COMPONENT_Y) + BsSet(2, COMPONENT_Cb) + BsSet(2, COMPONENT_Cr));
655
656
  }

657
658
  const TransformUnit& tuQ = *cuQ.cs->getTU(posQ, cuQ.chType);
  const TransformUnit& tuP = *cuP.cs->getTU(posP, cuP.chType);
659
  const PreCalcValues& pcv = *cu.cs->pcv;
Brian Heng's avatar
Brian Heng committed
660
  const unsigned rasterIdx = getRasterIdx( Position{ localPos.x,  localPos.y }, pcv );
661
662
  if (m_aapucBS[edgeDir][rasterIdx] && (cuP.firstPU->mhIntraFlag || cuQ.firstPU->mhIntraFlag))
  {
663
     return (BsSet(2, COMPONENT_Y) + BsSet(2, COMPONENT_Cb) + BsSet(2, COMPONENT_Cr));
664
  }
665

Kiran Misra's avatar
Kiran Misra committed
666
667
668
669
670
  unsigned tmpBs = 0;
  //-- Set BS for not Intra MB : BS = 2 or 1 or 0
  // Y
  if (m_aapucBS[edgeDir][rasterIdx] && (TU::getCbf(tuQ, COMPONENT_Y) || TU::getCbf(tuP, COMPONENT_Y)))
  {
671
    tmpBs += BsSet(1, COMPONENT_Y);
Kiran Misra's avatar
Kiran Misra committed
672
673
  }
  // U
674
675
676
#if JVET_O0105_ICT
  if (m_aapucBS[edgeDir][rasterIdx] && (TU::getCbf(tuQ, COMPONENT_Cb) || TU::getCbf(tuP, COMPONENT_Cb) || tuQ.jointCbCr || tuP.jointCbCr))
#else
Kiran Misra's avatar
Kiran Misra committed
677
  if (m_aapucBS[edgeDir][rasterIdx] && (TU::getCbf(tuQ, COMPONENT_Cb) || TU::getCbf(tuP, COMPONENT_Cb)))
678
#endif
Kiran Misra's avatar
Kiran Misra committed
679
  {
680
    tmpBs += BsSet(1, COMPONENT_Cb);
Kiran Misra's avatar
Kiran Misra committed
681
682
  }
  // V
683
684
685
#if JVET_O0105_ICT
  if (m_aapucBS[edgeDir][rasterIdx] && (TU::getCbf(tuQ, COMPONENT_Cr) || TU::getCbf(tuP, COMPONENT_Cr) || tuQ.jointCbCr || tuP.jointCbCr))
#else
Kiran Misra's avatar
Kiran Misra committed
686
  if (m_aapucBS[edgeDir][rasterIdx] && (TU::getCbf(tuQ, COMPONENT_Cr) || TU::getCbf(tuP, COMPONENT_Cr)))
687
#endif
Kiran Misra's avatar
Kiran Misra committed
688
  {
689
    tmpBs += BsSet(1, COMPONENT_Cr);
Kiran Misra's avatar
Kiran Misra committed
690
  }
691
  if (BsGet(tmpBs, COMPONENT_Y) == 1)
Kiran Misra's avatar
Kiran Misra committed
692
693
694
  {
    return tmpBs;
  }
695
696
697
698
  if ((cuP.firstPU->mhIntraFlag || cuQ.firstPU->mhIntraFlag))
  {
    return 1;
  }
699
700
701
702
703
704

  if ( !cu.Y().valid() )
  {
    return tmpBs;
  }

705
  // and now the pred
706
707
708
709
  const Position& lumaPosQ  = Position{ localPos.x,  localPos.y };
  const Position  lumaPosP  = ( edgeDir == EDGE_VER ) ? lumaPosQ.offset( -1, 0 ) : lumaPosQ.offset( 0, -1 );
  const MotionInfo&     miQ = cuQ.cs->getMotionInfo( lumaPosQ );
  const MotionInfo&     miP = cuP.cs->getMotionInfo( lumaPosP );
710
711
712
713
  const Slice&       sliceP = *cuP.slice;

  if (sliceQ.isInterB() || sliceP.isInterB())
  {
Yu Han's avatar
Yu Han committed
714
715
716
717
    const Picture *piRefP0 = (CU::isIBC(cuP) ? sliceP.getPic() : ((0 > miP.refIdx[0]) ? NULL : sliceP.getRefPic(REF_PIC_LIST_0, miP.refIdx[0])));
    const Picture *piRefP1 = (CU::isIBC(cuP) ? NULL            : ((0 > miP.refIdx[1]) ? NULL : sliceP.getRefPic(REF_PIC_LIST_1, miP.refIdx[1])));
    const Picture *piRefQ0 = (CU::isIBC(cuQ) ? sliceQ.getPic() : ((0 > miQ.refIdx[0]) ? NULL : sliceQ.getRefPic(REF_PIC_LIST_0, miQ.refIdx[0])));
    const Picture *piRefQ1 = (CU::isIBC(cuQ) ? NULL            : ((0 > miQ.refIdx[1]) ? NULL : sliceQ.getRefPic(REF_PIC_LIST_1, miQ.refIdx[1])));
718
719
720
721
722
723
724
    Mv mvP0, mvP1, mvQ0, mvQ1;

    if( 0 <= miP.refIdx[0] ) { mvP0 = miP.mv[0]; }
    if( 0 <= miP.refIdx[1] ) { mvP1 = miP.mv[1]; }
    if( 0 <= miQ.refIdx[0] ) { mvQ0 = miQ.mv[0]; }
    if( 0 <= miQ.refIdx[1] ) { mvQ1 = miQ.mv[1]; }

725
    int nThreshold = 1 << MV_FRACTIONAL_BITS_INTERNAL;
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
    unsigned uiBs = 0;

    //th can be optimized
    if ( ((piRefP0==piRefQ0)&&(piRefP1==piRefQ1)) || ((piRefP0==piRefQ1)&&(piRefP1==piRefQ0)) )
    {
      if ( piRefP0 != piRefP1 )   // Different L0 & L1
      {
        if ( piRefP0 == piRefQ0 )
        {
          uiBs  = ((abs(mvQ0.getHor() - mvP0.getHor()) >= nThreshold) || (abs(mvQ0.getVer() - mvP0.getVer()) >= nThreshold) ||
                   (abs(mvQ1.getHor() - mvP1.getHor()) >= nThreshold) || (abs(mvQ1.getVer() - mvP1.getVer()) >= nThreshold))
                  ? 1 : 0;
        }
        else
        {
          uiBs  = ((abs(mvQ1.getHor() - mvP0.getHor()) >= nThreshold) || (abs(mvQ1.getVer() - mvP0.getVer()) >= nThreshold) ||
                   (abs(mvQ0.getHor() - mvP1.getHor()) >= nThreshold) || (abs(mvQ0.getVer() - mvP1.getVer()) >= nThreshold))
                  ? 1 : 0;
        }
      }
      else    // Same L0 & L1
      {
        uiBs  = ((abs(mvQ0.getHor() - mvP0.getHor()) >= nThreshold) || (abs(mvQ0.getVer() - mvP0.getVer()) >= nThreshold) ||
                 (abs(mvQ1.getHor() - mvP1.getHor()) >= nThreshold) || (abs(mvQ1.getVer() - mvP1.getVer()) >= nThreshold))
              &&
                ((abs(mvQ1.getHor() - mvP0.getHor()) >= nThreshold) || (abs(mvQ1.getVer() - mvP0.getVer()) >= nThreshold) ||
                 (abs(mvQ0.getHor() - mvP1.getHor()) >= nThreshold) || (abs(mvQ0.getVer() - mvP1.getVer()) >= nThreshold))
              ? 1 : 0;
      }
    }
    else // for all different Ref_Idx
    {
      uiBs = 1;
    }
Kiran Misra's avatar
Kiran Misra committed
760
    return uiBs + tmpBs;
761
762
763
764
  }


  // pcSlice->isInterP()
Yu Han's avatar
Yu Han committed
765
766
767
768
  CHECK(CU::isInter(cuP) && 0 > miP.refIdx[0], "Invalid reference picture list index");
  CHECK(CU::isInter(cuP) && 0 > miQ.refIdx[0], "Invalid reference picture list index");
  const Picture *piRefP0 = (CU::isIBC(cuP) ? sliceP.getPic() : sliceP.getRefPic(REF_PIC_LIST_0, miP.refIdx[0]));
  const Picture *piRefQ0 = (CU::isIBC(cuQ) ? sliceQ.getPic() : sliceQ.getRefPic(REF_PIC_LIST_0, miQ.refIdx[0]));
769
770
  if (piRefP0 != piRefQ0)
  {
Kiran Misra's avatar
Kiran Misra committed
771
    return tmpBs + 1;
772
773
774
775
776
  }

  Mv mvP0 = miP.mv[0];
  Mv mvQ0 = miQ.mv[0];

777
  int nThreshold = 1 << MV_FRACTIONAL_BITS_INTERNAL;
Kiran Misra's avatar
Kiran Misra committed
778
  return ( ( abs( mvQ0.getHor() - mvP0.getHor() ) >= nThreshold ) || ( abs( mvQ0.getVer() - mvP0.getVer() ) >= nThreshold ) ) ? (tmpBs + 1) : tmpBs;
779
780
}

781
#if LUMA_ADAPTIVE_DEBLOCKING_FILTER_QP_OFFSET
782
void LoopFilter::deriveLADFShift( const Pel* src, const int stride, int& shift, const DeblockEdgeDir edgeDir, const SPS sps )
783
{
784
  uint32_t lumaLevel = 0;
785
  shift = sps.getLadfQpOffset(0);
786
787
788

  if (edgeDir == EDGE_VER)
  {
789
    lumaLevel = (src[0] + src[3*stride] + src[-1] + src[3*stride - 1]) >> 2;
790
791
792
  }
  else // (edgeDir == EDGE_HOR)
  {
793
    lumaLevel = (src[0] + src[3] + src[-stride] + src[-stride + 3]) >> 2;
794
795
  }

796
  for ( int k = 1; k < sps.getLadfNumIntervals(); k++ )
797
  {
798
    const int th = sps.getLadfIntervalLowerBound( k );
799
    if ( lumaLevel > th )
800
    {
801
      shift = sps.getLadfQpOffset( k );
802
803
804
805
806
807
808
809
810
    }
    else
    {
      break;
    }
  }
}
#endif

811
void LoopFilter::xEdgeFilterLuma( const CodingUnit& cu, const DeblockEdgeDir edgeDir, const int iEdge )
812
813
814
815
{
  const CompArea&  lumaArea = cu.block(COMPONENT_Y);
  const PreCalcValues& pcv = *cu.cs->pcv;

Nan Hu's avatar
Nan Hu committed
816
  PelBuf        picYuvRec = m_enc ? m_encPicYuvBuffer.getBuf( lumaArea ) : cu.cs->getRecoBuf( lumaArea );
817
818
819
820
821
822
823
  Pel           *piSrc    = picYuvRec.buf;
  const int     iStride   = picYuvRec.stride;
  Pel           *piTmpSrc = piSrc;
  const PPS     &pps      = *(cu.cs->pps);
  const SPS     &sps      = *(cu.cs->sps);
  const Slice   &slice    = *(cu.slice);
  const bool    ppsTransquantBypassEnabledFlag = pps.getTransquantBypassEnabledFlag();
824
825
826
#if JVET_O0119_BASE_PALETTE_444
  const bool    spsPaletteEnabledFlag = sps.getPLTMode();
#endif
827
828
829
830
  const int     bitDepthLuma                   = sps.getBitDepth(CHANNEL_TYPE_LUMA);
  const ClpRng& clpRng( cu.cs->slice->clpRng(COMPONENT_Y) );

  int          iQP          = 0;
Karsten Suehring's avatar
Karsten Suehring committed
831
  unsigned     uiNumParts   = ( ( ( edgeDir == EDGE_VER ) ? lumaArea.height / pcv.minCUHeight : lumaArea.width / pcv.minCUWidth ) );
832
833
834
835
  int          pelsInPart   = pcv.minCUWidth;
  unsigned     uiBsAbsIdx   = 0, uiBs = 0;
  int          iOffset, iSrcStep;

836
  bool  bPCMFilter      = (sps.getPCMEnabledFlag() && sps.getPCMFilterDisableFlag()) ? true : false;
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
  bool  bPartPNoFilter  = false;
  bool  bPartQNoFilter  = false;
  int   betaOffsetDiv2  = slice.getDeblockingFilterBetaOffsetDiv2();
  int   tcOffsetDiv2    = slice.getDeblockingFilterTcOffsetDiv2();
  int   xoffset, yoffset;

  Position pos;

  if (edgeDir == EDGE_VER)
  {
    xoffset   = 0;
    yoffset   = pelsInPart;
    iOffset   = 1;
    iSrcStep  = iStride;
    piTmpSrc += iEdge * pelsInPart;
    pos       = Position{ lumaArea.x + iEdge * pelsInPart, lumaArea.y - yoffset };
  }
  else  // (edgeDir == EDGE_HOR)
  {
    xoffset   = pelsInPart;
    yoffset   = 0;
    iOffset   = iStride;
    iSrcStep  = 1;
    piTmpSrc += iEdge*pelsInPart*iStride;
    pos       = Position{ lumaArea.x - xoffset, lumaArea.y + iEdge * pelsInPart };
  }

  const int iBitdepthScale = 1 << (bitDepthLuma - 8);

  // dec pos since within the loop we first calc the pos
  for( int iIdx = 0; iIdx < uiNumParts; iIdx++ )
  {
    pos.x += xoffset;
    pos.y += yoffset;

872
873
874
875
876
877
878
879
880
881
    // Deblock luma boundaries on 8x8 grid only
    if ( edgeDir == EDGE_HOR && ( pos.y % 8 ) != 0 )
    {
      continue;
    }
    if ( edgeDir == EDGE_VER && ( pos.x % 8 ) != 0 )
    {
      continue;
    }

882
    uiBsAbsIdx = getRasterIdx( pos, pcv );
883
    uiBs = BsGet(m_aapucBS[edgeDir][uiBsAbsIdx], COMPONENT_Y);
884
885
886
887
888
889
890
891

    if( uiBs )
    {
      const CodingUnit& cuQ =  cu;
      const CodingUnit& cuP = *cu.cs->getCU(pos.offset(xoffset - pelsInPart, yoffset - pelsInPart), cu.chType);
      // Derive neighboring PU index
      if (edgeDir == EDGE_VER)
      {
Karsten Suehring's avatar
Karsten Suehring committed
892
        CHECK( !isAvailableLeft( cu, cuP, !slice.getLFCrossSliceBoundaryFlag(), !pps.getLoopFilterAcrossBricksEnabledFlag() ), "Neighbour not available" );
893
894
895
      }
      else  // (iDir == EDGE_HOR)
      {
Karsten Suehring's avatar
Karsten Suehring committed
896
        CHECK( !isAvailableAbove( cu, cuP, !slice.getLFCrossSliceBoundaryFlag(), !pps.getLoopFilterAcrossBricksEnabledFlag() ), "Neighbour not available" );
897
898
899
900
      }

      iQP = (cuP.qp + cuQ.qp + 1) >> 1;

901
#if LUMA_ADAPTIVE_DEBLOCKING_FILTER_QP_OFFSET
902
      if ( sps.getLadfEnabled() )
903
904
905
906
907
908
      {
        int iShift = 0;
        deriveLADFShift( piTmpSrc + iSrcStep * (iIdx*pelsInPart), iStride, iShift, edgeDir, sps );
        iQP += iShift;
      }
#endif
Kiran Misra's avatar
Kiran Misra committed
909
910
911

      bool sidePisLarge   = false;
      bool sideQisLarge   = false;
912
913
      int maxFilterLengthP = m_maxFilterLengthP[COMPONENT_Y][pos.x-m_ctuXLumaSamples][pos.y-m_ctuYLumaSamples];
      int maxFilterLengthQ = m_maxFilterLengthQ[COMPONENT_Y][pos.x-m_ctuXLumaSamples][pos.y-m_ctuYLumaSamples];
Kiran Misra's avatar
Kiran Misra committed
914
915
      if (maxFilterLengthP > 3)
      {
916
917
        sidePisLarge = true;
        if ( maxFilterLengthP > 5 )
Kiran Misra's avatar
Kiran Misra committed
918
919
        {
          // restrict filter length if sub-blocks are used (e.g affine or ATMVP)
Luong Pham Van's avatar
Luong Pham Van committed
920
          if (cuP.affine)
Kiran Misra's avatar
Kiran Misra committed
921
922
923
924
925
926
927
          {
            maxFilterLengthP = std::min(maxFilterLengthP, 5);
          }
        }
      }
      if (maxFilterLengthQ > 3)
      {
928
        sideQisLarge = true;
Kiran Misra's avatar
Kiran Misra committed
929
930
931
932
933
934
      }

      if (edgeDir == EDGE_HOR && pos.y % slice.getSPS()->getCTUSize() == 0)
      {
        sidePisLarge = false;
      }
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
      const int iIndexTC  = Clip3(0, MAX_QP + DEFAULT_INTRA_TC_OFFSET, int(iQP + DEFAULT_INTRA_TC_OFFSET*(uiBs - 1) + (tcOffsetDiv2 << 1)));
      const int iIndexB   = Clip3(0, MAX_QP, iQP + (betaOffsetDiv2 << 1));

      const int iTc       = sm_tcTable  [iIndexTC] * iBitdepthScale;
      const int iBeta     = sm_betaTable[iIndexB ] * iBitdepthScale;
      const int iSideThreshold = ( iBeta + ( iBeta >> 1 ) ) >> 3;
      const int iThrCut   = iTc * 10;

      const unsigned uiBlocksInPart = pelsInPart / 4 ? pelsInPart / 4 : 1;

      for( int iBlkIdx = 0; iBlkIdx < uiBlocksInPart; iBlkIdx++ )
      {
        const int dp0 = xCalcDP(piTmpSrc + iSrcStep*(iIdx*pelsInPart + iBlkIdx * 4 + 0), iOffset);
        const int dq0 = xCalcDQ(piTmpSrc + iSrcStep*(iIdx*pelsInPart + iBlkIdx * 4 + 0), iOffset);
        const int dp3 = xCalcDP(piTmpSrc + iSrcStep*(iIdx*pelsInPart + iBlkIdx * 4 + 3), iOffset);
        const int dq3 = xCalcDQ(piTmpSrc + iSrcStep*(iIdx*pelsInPart + iBlkIdx * 4 + 3), iOffset);
Kiran Misra's avatar
Kiran Misra committed
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
        int dp0L = dp0;
        int dq0L = dq0;
        int dp3L = dp3;
        int dq3L = dq3;

        if (sidePisLarge)
        {
          dp0L = (dp0L + xCalcDP(piTmpSrc + iSrcStep*(iIdx*pelsInPart + iBlkIdx * 4 + 0) - 3 * iOffset, iOffset) + 1) >> 1;
          dp3L = (dp3L + xCalcDP(piTmpSrc + iSrcStep*(iIdx*pelsInPart + iBlkIdx * 4 + 3) - 3 * iOffset, iOffset) + 1) >> 1;
        }
        if (sideQisLarge)
        {
          dq0L = (dq0L + xCalcDQ(piTmpSrc + iSrcStep*(iIdx*pelsInPart + iBlkIdx * 4 + 0) + 3 * iOffset, iOffset) + 1) >> 1;
          dq3L = (dq3L + xCalcDQ(piTmpSrc + iSrcStep*(iIdx*pelsInPart + iBlkIdx * 4 + 3) + 3 * iOffset, iOffset) + 1) >> 1;
        }
        bool useLongtapFilter = false;
        if (sidePisLarge || sideQisLarge)
        {
          int d0L = dp0L + dq0L;
          int d3L = dp3L + dq3L;

          int dpL = dp0L + dp3L;
          int dqL = dq0L + dq3L;

          int dL = d0L + d3L;

          bPartPNoFilter = bPartQNoFilter = false;
          if (bPCMFilter)
          {
            // Check if each of PUs is I_PCM with LF disabling
            bPartPNoFilter = cuP.ipcm;
            bPartQNoFilter = cuQ.ipcm;
          }
          if (ppsTransquantBypassEnabledFlag)
          {
            // check if each of PUs is lossless coded
            bPartPNoFilter = bPartPNoFilter || cuP.transQuantBypass;
            bPartQNoFilter = bPartQNoFilter || cuQ.transQuantBypass;
          }
990
991
992
993
994
995
996
997
#if JVET_O0119_BASE_PALETTE_444
		  if (spsPaletteEnabledFlag)
		  {
			// check if each of PUs is palette coded
			bPartPNoFilter = bPartPNoFilter || CU::isPLT(cuP);
			bPartQNoFilter = bPartQNoFilter || CU::isPLT(cuQ);
		  }
#endif
Kiran Misra's avatar
Kiran Misra committed
998
999
1000

          if (dL < iBeta)
          {