doc/software-manual.tex

@@ -2175,6 +2175,11 @@ For Cb component with BT.2020 container use 1.14; for BT.709 material and 1.04 f
 For Cr component with BT.2020 container use 1.79; for BT.709 material and 1.39 for P3 material.
 \\
 
+\Option{BIM} &
+\Default{false} &
+Enable or disable Block Importance Mapping, QP adaptation depending on estimated propagation of reference samples. Depends on future and past reference frames configured for temporal filter.
+\\
+
 \Option{SliceChromaQPOffsetPeriodicity} &
 \Default{0} &
 Defines the periodicity for inter slices that use the slice-level chroma QP offsets, as defined by SliceCbQpOffsetIntraOrPeriodic and SliceCrQpOffsetIntraOrPeriodic. A value of 0 disables the periodicity. It is intended to be used in low-delay configurations where an regular intra period is not defined.

script/decodeAndEstimateRate.py

+#!/usr/bin/python
+
+import sys, os, subprocess
+
+if len( sys.argv ) < 3:
+    print( "Usage: decodeAndEstimateRate_parallel.py exePath bitstreamFile outputDirectory" )
+    sys.exit()
+
+if not os.path.isfile( sys.argv[2] ):
+    print( "Error: Could not open bitstreamFile: '" + sys.argv[2] + "'" )
+    sys.exit(-1)
+
+workDir = os.path.abspath( os.getcwd() )
+outDir = sys.argv[3]
+binDir = sys.argv[1]
+
+if not os.path.isdir( outDir ):
+    print( "Error: outputDirectory doesn't exist." )
+    sys.exit(-1)
+
+xmlFile = "CabacBits_data.xml"
+binFile = "CabacBits_data.bin"
+
+if os.path.isfile( os.path.join(binDir, xmlFile) ):
+    print( "Error: CabacBits_data.xml already exists." )
+    sys.exit(-1)
+
+if os.path.isfile( os.path.join(binDir, binFile) ):
+    print( "Error: CabacBits_data.bin already exists." )
+    sys.exit(-1)
+
+logDir = sys.argv[2].strip(".bin") + ".log"
+f = open(logDir, "w") 
+
+subprocess.call( [ os.path.join( binDir, 'DecoderAppStatic' ), '-b', sys.argv[2] ], cwd=binDir, stdout=f )
+
+command = os.path.join( binDir, 'RateEstimatorStatic' )
+command += " " + os.path.join( binDir, xmlFile) + " " + os.path.join( binDir, binFile) + " " + outDir
+os.system(command)
+
+os.remove( os.path.join(binDir, xmlFile) )
+os.remove( os.path.join(binDir, binFile) )
+
+
+
+

script/example.sh

+#!/bin/sh
+
+SCRIPTPATH=$(pwd)
+echo $SCRIPTPATH
+
+cd $SCRIPTPATH/str/
+bins=(*.bin)                 
+cd ../
+
+for bin in "${bins[@]}"; do
+  file=$(echo "${bin}" | cut -d "." -f 1)
+  cd info
+  mkdir $file
+  cd ../
+
+  python decodeAndEstimateRate.py $SCRIPTPATH $SCRIPTPATH/str/$bin $SCRIPTPATH/info/$file
+  echo "$file done"
+done
+
+python selectParameters.py $SCRIPTPATH/info
\ No newline at end of file

script/selectParameters.py

+#!/usr/bin/python
+
+from __future__ import division
+from __future__ import print_function
+
+import sys, os
+#from __builtin__ import False
+
+BITS_SHIFT = 30
+
+shiftIdxList = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13 ]
+rateIdxList = [ 4, 11, 18, 25, 32]
+
+if len( sys.argv ) != 2:
+    print( "Usage: selectParameters.py ratesDirectory" )
+    sys.exit()
+
+if not os.path.isdir( sys.argv[1] ):
+    print( "Error: Could not access ratesDirectory: '" + sys.argv[2] + "'" )
+    sys.exit(-1)
+
+
+
+def checkSeqFoldersAndGetCtxList( sfPath, sf ):
+    if len( sf ) == 0:
+        print( "Error: Could not find any seqence folders." )
+        sys.exit(-1)
+
+    print( "Checking consistency of 'CabacBits_data_index.txt' files in %d folders..." % len( sf ) )
+
+    firstSeqLines = None
+    seqBits = []
+    for s in sf:
+        idxFile = os.path.join( sfPath, s, "CabacBits_data_index.txt" )
+        if not os.path.isfile( idxFile ):
+            print( "Error: Could not access index file: '" + idxFile + "'" )
+            sys.exit(-1)
+        with open( idxFile, "rt" ) as f:
+            sfLines = [ x.rstrip() for x in f.readlines() ]
+            seqBits.append( int( sfLines[0] ) )
+            sfLines = sfLines[1:]
+        if firstSeqLines is None:
+            firstSeqLines = sfLines
+        else:
+            if sfLines != firstSeqLines:
+                print( "Error: CabacBits_data_index.txt file in folder '%s' doesn't contain the same list of ctx files in the same order as in folder '%s'." % (s, sf[0]) )
+                sys.exit(-1)
+    print( "Success." )
+    return firstSeqLines, seqBits
+
+
+seqFolders = [x for x in os.listdir( sys.argv[1] ) if os.path.isdir( os.path.join( sys.argv[1], x ) ) ]
+
+ctxList, seqBits = checkSeqFoldersAndGetCtxList( sys.argv[1], seqFolders )
+
+def parseCtxFile( ctxFileName, numInits, sequenceBits, bits ):
+    try:
+        with open( ctxFileName, "rt" ) as f:
+            ctx = [ x.rstrip() for x in f.readlines() ]
+    except:
+        print( "Could not access ctx file: %s", f )
+        sys.exit(-1)
+    runIdx = 0
+    initBitsList = False
+    if len( bits ) == 0:
+        initBitsList = True
+    for initType in range( 3 ):
+        if initBitsList:
+            bits.append([])
+        for ws in range( 13 ):
+            if initBitsList:
+                bits[initType].append([])
+            for adaRate in range (5):            
+                if initBitsList:
+                    bits[initType][ws].append([])
+            
+                initBits = [x for x in ctx[runIdx].split( ',' ) ]
+                if initBits[-1].strip() == '':
+                    initBits = initBits[:-1]
+                initBits = [int(x) for x in initBits ]
+            
+                if numInits is None:
+                    numInits = len( initBits )
+                    print( "Found %d init value candidates." % numInits )
+                else:
+                    if numInits != len( initBits ):
+                        print( "Error: File '%s' seems inconsistent." % ctxFileName )
+                        sys.exit(-1)
+        
+                if initBitsList:
+                    bits[initType][ws][adaRate] = [(x << (BITS_SHIFT-15)) // sequenceBits for x in initBits]
+                else:
+                    bits[initType][ws][adaRate] = [bits[initType][ws][adaRate][i] + (x << (BITS_SHIFT-15)) // sequenceBits for i, x in enumerate(initBits)]
+                runIdx += 1
+
+        if initType < 2 and ctx[runIdx] != "":
+            print( "Unexpected nonempty line between two blocks of bit rates." )
+            sys.exit(-1)
+        else:
+            runIdx += 1
+    offset = [x for x in ctx[runIdx].split( ',' ) ]            
+    offset0 = int(offset[0])
+    offset1 = int(offset[1])
+    initB   = int(offset[2])
+    rateB   = int(offset[3])
+    weightB = int(offset[4])
+    initP   = int(offset[5])
+    rateP   = int(offset[6])
+    weightP = int(offset[7])
+    initI   = int(offset[8])
+    rateI   = int(offset[9])
+    weightI = int(offset[10])
+    return numInits, bits, offset0, offset1, initB, rateB, weightB, initP, rateP, weightP, initI, rateI, weightI
+
+
+lastCtxName = None
+ctxArray = []
+allCtx = []
+
+numInits = None
+for ctxFile in ctxList:
+    ctxBits = []
+    #input()
+    for i, seq in enumerate( seqFolders ):
+        ctxFn = os.path.join( sys.argv[1], seq, ctxFile )
+        numInits, ctxBits, offset0, offset1, initB, rateB, weightB, initP, rateP, weightP, initI, rateI, weightI = parseCtxFile( ctxFn, numInits, seqBits[i], ctxBits )
+    print( "Processing %s" % ctxFile )
+    #print("ctxBits ", ctxBits)
+    
+    # create new array
+    _, _, ctxName, _ = ctxFile.split( "_" )
+    if ctxName != lastCtxName and lastCtxName is not None:
+        allCtx.append( [ lastCtxName, zip( *ctxArray ) ] )
+        ctxArray = []
+    
+    lastCtxName = ctxName
+    
+    bestBits = 0
+    bestLine = []
+    minInit = []
+    minInitIdx = []
+    for sliceTypeIdx, sliceType in enumerate(ctxBits):
+        minInit.append([])
+        minInitIdx.append([])
+        for wsIdx, ws in enumerate(sliceType):
+            minInit[sliceTypeIdx].append([])
+            minInitIdx[sliceTypeIdx].append([])
+            for adaRateIdx, adaRate in enumerate(ws):
+                minInit[sliceTypeIdx][wsIdx].append(min(adaRate))
+                minInitIdx[sliceTypeIdx][wsIdx].append( adaRate.index( min(adaRate) ) )
+        #print("\n")
+        #print("minInit", minInit)
+        #print("\n")
+        #print("minInitIdx", minInitIdx)
+        
+        # decide adaptive rate for each window size
+        minAdaRate = []
+        minAdaRateIdx = []
+        for sliceTypeIdx, sliceType in enumerate( minInit ):
+            minAdaRate.append([])
+            minAdaRateIdx.append([])
+            for wsIdx, ws in enumerate(sliceType):
+                minAdaRate[sliceTypeIdx].append( min(ws) )
+                minAdaRateIdx[sliceTypeIdx].append( ws.index( min(ws) ) )
+        #print("\n")
+        #print("minAdaRate", minAdaRate)
+        #print("\n")
+        #print("minAdaRateIdx", minAdaRateIdx)
+        
+        # decide window size for each sliceType
+        currLine = []
+        totalBits = 0
+        for sliceTypeIdx, sliceType in enumerate( minAdaRate ):
+            f_minWs = min( sliceType )
+            f_minWsIdx = sliceType.index( f_minWs )
+            f_minAdaRateIdx = minAdaRateIdx[sliceTypeIdx][f_minWsIdx]
+            f_minInitIdx = minInitIdx[sliceTypeIdx][f_minWsIdx][f_minAdaRateIdx]
+            #print("f_minWs", f_minWs)
+            #print("f_minWsIdx", f_minWsIdx)
+            #print("f_minAdaRateIdx", f_minAdaRateIdx)
+            #print("f_minInitIdx", f_minInitIdx)
+            totalBits += f_minWs
+            if sliceTypeIdx == 0:
+                if f_minWs <= 0:
+                    currLine.append(-1)
+                    currLine.append(-1)
+                    currLine.append(-1)
+                else:
+                    currLine.append( f_minInitIdx )
+                    currLine.append( shiftIdxList[f_minWsIdx] )
+                    currLine.append( rateIdxList[f_minAdaRateIdx] )
+            elif sliceTypeIdx == 1:
+                currLine.append(initP)
+                currLine.append(rateP)
+                currLine.append(weightP)
+            elif sliceTypeIdx == 2:
+                currLine.append(initB)
+                currLine.append(rateB)
+                currLine.append(weightB)
+        currLine.append( offset0 )
+        currLine.append( offset1 )
+    
+        #print("\n")
+        #print("currLine", currLine)
+        #print("totalBits", totalBits)
+    #print("\n")
+    #print("bestLine", bestLine)
+    #print("bestBits", bestBits)
+    ctxArray.append( currLine )
+        
+    if ctxFile == ctxList[-1]:
+        allCtx.append( [ lastCtxName, zip( *ctxArray ) ] )
+
+
+indent = ""
+for name, [ I, WS_I, ADA_I, P, WS_P, ADA_P, B, WS_B, ADA_B, OFF0, OFF1 ] in allCtx:
+    I  = "".join( [ "%3d, " % x if x >= 0 else "CNU, " for x in I ] )
+    P  = "".join( [ "%3d, " % x if x >= 0 else "CNU, " for x in P ] )
+    B  = "".join( [ "%3d, " % x if x >= 0 else "CNU, " for x in B ] )
+    WS_I = "".join( [ "%3d, " % x if x >= 0 else "DWS, " for x in WS_I ] )
+    WS_P = "".join( [ "%3d, " % x if x >= 0 else "DWS, " for x in WS_P ] )
+    WS_B = "".join( [ "%3d, " % x if x >= 0 else "DWS, " for x in WS_B ] )
+    ADA_I = "".join( [ "%3d, " % x if x >= 0 else "DWE, " for x in ADA_I ] )
+    ADA_P = "".join( [ "%3d, " % x if x >= 0 else "DWE, " for x in ADA_P ] )
+    ADA_B = "".join( [ "%3d, " % x if x >= 0 else "DWE, " for x in ADA_B ] )
+    OFF0 = "".join( [ "%3d, " % x if x >= 0 else "DWO, " for x in OFF0 ] )
+    OFF1 = "".join( [ "%3d, " % x if x >= 0 else "DWO, " for x in OFF1 ] )
+    nameParts = name.split( "[" )
+    if len( nameParts ) > 1:
+        if( nameParts[1] == "0]" ):
+            if indent != "":
+                print( "};\n")
+            print( "const CtxSet ContextSetCfg::%s[] = " % nameParts[0] )
+            print( "{" )
+        indent = "  "
+        print( "%sContextSetCfg::addCtxSet" % indent )
+    else:
+        if indent != "":
+            print( "};\n")
+        indent = ""
+        print( "const CtxSet ContextSetCfg::%s = ContextSetCfg::addCtxSet" % name )
+    print( "%s({" % indent );
+    print( "%s  { %s}," % (indent, B ) )
+    print( "%s  { %s}," % (indent, P ) )
+    print( "%s  { %s}," % (indent, I ) )
+    print( "%s  { %s}," % (indent, WS_B) )
+    print( "%s  { %s}," % (indent, WS_P) )
+    print( "%s  { %s}," % (indent, WS_I) )
+    print( "%s  { %s}," % (indent, ADA_B) )
+    print( "%s  { %s}," % (indent, ADA_P) )
+    print( "%s  { %s}," % (indent, ADA_I) )
+    print( "%s  { %s}," % (indent, OFF0) )
+    print( "%s  { %s}," % (indent, OFF1) )
+    #print( "%s  { %s}," % (indent, WS) )
+    print( "%s})%s" % ( indent, ";\n" if indent == "" else "," ) )
+if indent != "":
+    print( "};\n")
+
+
+
+

source/App/BitstreamExtractorApp/BitstreamExtractorApp.cpp

@@ -3,7 +3,7 @@
  * and contributor rights, including patent rights, and no such rights are
  * granted under this license.
  *
- * Copyright (c) 2010-2020, ITU/ISO/IEC
+ * Copyright (c) 2010-2023, ITU/ISO/IEC
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -505,7 +505,6 @@ uint32_t BitstreamExtractorApp::decode()
   bitstreamFileIn.clear();
   bitstreamFileIn.seekg( 0, std::ios::beg );
 
-  int unitCnt = 0;
   bool lastSliceWritten= false;   // stores status of previous slice for associated filler data NAL units
 
   VPS *vpsIdZero = new VPS();
@@ -802,7 +801,6 @@ uint32_t BitstreamExtractorApp::decode()
           writeInpuNalUnitToStream = lastSliceWritten;
         }
       }
-      unitCnt++;
 
       if( writeInpuNalUnitToStream )
       {

source/App/BitstreamExtractorApp/BitstreamExtractorApp.h

@@ -3,7 +3,7 @@
  * and contributor rights, including patent rights, and no such rights are
  * granted under this license.
  *
- * Copyright (c) 2010-2020, ITU/ISO/IEC
+ * Copyright (c) 2010-2023, ITU/ISO/IEC
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

source/App/BitstreamExtractorApp/BitstreamExtractorAppCfg.cpp

@@ -3,7 +3,7 @@
  * and contributor rights, including patent rights, and no such rights are
  * granted under this license.
  *
- * Copyright (c) 2010-2020, ITU/ISO/IEC
+ * Copyright (c) 2010-2023, ITU/ISO/IEC
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

source/App/BitstreamExtractorApp/BitstreamExtractorAppCfg.h

@@ -3,7 +3,7 @@
  * and contributor rights, including patent rights, and no such rights are
  * granted under this license.
  *
- * Copyright (c) 2010-2020, ITU/ISO/IEC
+ * Copyright (c) 2010-2023, ITU/ISO/IEC
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

source/App/BitstreamExtractorApp/CMakeLists.txt

@@ -25,7 +25,7 @@ endif()
 add_executable( ${EXE_NAME} ${SRC_FILES} ${INC_FILES} ${NATVIS_FILES} )
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-if( SET_ENABLE_TRACING )
+if( DEFINED ENABLE_TRACING )
   if( ENABLE_TRACING )
     target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_TRACING=1 )
   else()
@@ -33,24 +33,12 @@ if( SET_ENABLE_TRACING )
   endif()
 endif()
 
-if( OpenMP_FOUND )
-  if( SET_ENABLE_SPLIT_PARALLELISM )
-    if( ENABLE_SPLIT_PARALLELISM )
-      target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_SPLIT_PARALLELISM=1 )
-    else()
-      target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_SPLIT_PARALLELISM=0 )
-    endif()
-  endif()
-  if( SET_ENABLE_WPP_PARALLELISM )
-    if( ENABLE_WPP_PARALLELISM )
-      target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_WPP_PARALLELISM=1 )
-    else()
-      target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_WPP_PARALLELISM=0 )
-    endif()
+if( DEFINED ENABLE_HIGH_BITDEPTH )
+  if( ENABLE_HIGH_BITDEPTH )
+    target_compile_definitions( ${EXE_NAME} PUBLIC RExt__HIGH_BIT_DEPTH_SUPPORT=1 )
+  else()
+    target_compile_definitions( ${EXE_NAME} PUBLIC RExt__HIGH_BIT_DEPTH_SUPPORT=0 )
   endif()
-else()
-  target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_SPLIT_PARALLELISM=0 )
-  target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_WPP_PARALLELISM=0 )
 endif()
 
 if( CMAKE_COMPILER_IS_GNUCC AND BUILD_STATIC )
@@ -58,7 +46,7 @@ if( CMAKE_COMPILER_IS_GNUCC AND BUILD_STATIC )
   target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_WPP_STATIC_LINK=1 )
 endif()
 
-target_link_libraries( ${EXE_NAME} CommonLib EncoderLib DecoderLib Utilities Threads::Threads ${ADDITIONAL_LIBS} )
+target_link_libraries( ${EXE_NAME} CommonLib EncoderLib DecoderLib Utilities ${ADDITIONAL_LIBS} )
 
 # lldb custom data formatters
 if( XCODE )

source/App/BitstreamExtractorApp/bitstreamextractormain.cpp

@@ -3,7 +3,7 @@
  * and contributor rights, including patent rights, and no such rights are
  * granted under this license.
  *
- * Copyright (c) 2010-2020, ITU/ISO/IEC
+ * Copyright (c) 2010-2023, ITU/ISO/IEC
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -52,7 +52,7 @@ int main(int argc, char* argv[])
   fprintf( stdout, NVM_ONOS );
   fprintf( stdout, NVM_COMPILEDBY );
   fprintf( stdout, NVM_BITS );
-#if ENABLE_SIMD_OPT
+#if ENABLE_SIMD_OPT && defined(TARGET_SIMD_X86)
   std::string SIMD;
   df::program_options_lite::Options optsSimd;
   optsSimd.addOptions()( "SIMD", SIMD, std::string( "" ), "" );

source/App/CabacTraining/CMakeLists.txt

+# executable
+set( EXE_NAME CabacTraining )
+
+# get source files
+file( GLOB SRC_FILES "*.cpp" )
+
+# get include files
+file( GLOB INC_FILES "*.h" )
+
+# add executable
+add_executable( ${EXE_NAME} ${SRC_FILES} ${INC_FILES} )
+
+target_link_libraries( ${EXE_NAME} CommonLib Utilities ${ADDITIONAL_LIBS} )
+
+# include the output directory, where the svnrevision.h file is generated
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+include_directories(${CMAKE_SOURCE_DIR}/source/Lib)
+
+if( CMAKE_SYSTEM_NAME STREQUAL "Linux" )
+  add_custom_command( TARGET ${EXE_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy
+                                                          $<$<CONFIG:Debug>:${CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG}/CabacTraining>
+                                                          $<$<CONFIG:Release>:${CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE}/CabacTraining>
+                                                          $<$<CONFIG:RelWithDebInfo>:${CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO}/CabacTraining>
+                                                          $<$<CONFIG:MinSizeRel>:${CMAKE_RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL}/CabacTraining>
+                                                          $<$<CONFIG:Debug>:${CMAKE_SOURCE_DIR}/bin/CabacTrainingd>
+                                                          $<$<CONFIG:Release>:${CMAKE_SOURCE_DIR}/bin/CabacTraining>
+                                                          $<$<CONFIG:RelWithDebInfo>:${CMAKE_SOURCE_DIR}/bin/CabacTrainingp>
+                                                          $<$<CONFIG:MinSizeRel>:${CMAKE_SOURCE_DIR}/bin/CabacTrainingm> )
+endif()
+
+# set the folder where to place the projects
+set_target_properties( ${EXE_NAME}  PROPERTIES FOLDER app LINKER_LANGUAGE CXX )
+

source/App/CabacTraining/README.md

+# CABAC training
+
+## 1- Dump CABAC bins
+Run the decoder with the config parameter ActivateCABACDumping set equal to 1.
+
+For each configurations (RA, AI, LDB, LDP) create a directory and decode all bitstream. 
+Example:
+``sh
+mkdir -p AI;
+cd AI;
+for bs in /path/to/AI/*.bin; do
+ DecoderAppStatic -b $bs --ActivateCABACDumping=1
+``
+It will create N files (N is the total number of contexts) for each decoded bitstream.
+All bistreams can be decoded in parallel (the output filename is deduced from the the bitstream name).
+
+## 2- Train a context
+
+Build the CabacTrain application.
+Send all the file related to a context to the trainer.
+
+Example:
+``sh
+CTX=821;
+cat AI/*.cabac_${CTX} RA/*.cabac_${CTX} LDB/*.cabac_${CTX} LDP/*.cabac_${CTX} | cabacTrain --output=context_${CTX}.txt
+``
+
+The resulting parameters are in the file:
+``
+821 A 26 41 35 7 7 8 4 4 18 99 115 99 115 99 115
+``
+821 is the context number, ``A`` can be ignored. Following numbers are the initialization parameters.
+
+All contexts can be trained in parallel.
+
+## Troubleshooting
+
+If there is an error that the bin file is not opened, one of the reasons can be that the limit of opened files is not enough.
+In Linux, use ulimit to increase that number, for example, ulimit -n 2048 (assuming the total number of contexts is less than 2048).
\ No newline at end of file

source/App/CabacTraining/data.h

+#pragma once
+#include <vector>
+#include <array>
+#include "Contexts.h"
+
+using QP = int;
+using Bin = uint8_t;
+using Bins = std::vector<Bin>;
+
+// as read in the Contexts.cpp file
+struct ModelParameters
+{
+  int initId;
+  int log2windowsize;
+  int adaptweight;
+  int rateoffset[2];
+};
+
+#if JVET_AH0176_LOW_DELAY_B_CTX
+constexpr int kNbModels = 4;
+using Models = std::array<ModelParameters, kNbModels>;
+#endif
+
+struct DataFrame
+{
+  Bins bins;
+  SliceType type;
+  SliceType reportslice;
+  int poc;
+  bool tempCABAC;
+  bool switchBp;
+  QP qp;
+
+  // parameters as set internally inside cabac
+  uint16_t p0, p1;
+  int rate;
+  int weight;
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+  int drate0;
+  int drate1;
+#endif
+};
+
+struct DataSequence
+{
+  std::vector<DataFrame> v;
+  uint64_t filesize = 0;
+};
+
+using DataSequences = std::vector<DataSequence>;
+
+struct DataDb
+{
+  int ctxidx = -1;
+#if JVET_AH0176_LOW_DELAY_B_CTX
+  Models modelsBPI;
+#else
+  std::array<ModelParameters, 3> modelsBPI;
+#endif
+  DataSequences v;
+};
+
+inline int window2Coding( int w0, int w1 )
+{
+  int log2WindowSize = ((w0 & 3) << 2) | (w1 & 3); // stupid encoding
+  return log2WindowSize;
+}
+
+inline std::tuple<int, int> coding2Window( int rate )
+{
+  int rate0 = 2 + ((rate >> 2) & 3);
+  int rate1 = 3 + rate0 + (rate & 3);
+  return std::make_tuple( rate0, rate1 );
+}
+
+inline int alpha2Coding( int alpha0, int alpha1 )
+{
+  return (alpha0 << 3) | (alpha1 & 0x07);
+}
+
+inline  std::tuple<int, int> coding2alpha( int code )
+{
+  return std::make_tuple( code >> 3, code & 0x07 );
+}
+
+inline int drate2Coding( int dr0, int dr1 )
+{
+// for a bin 0 or 1
+//  int drate0=  (mi.rateoffset0>>4)-ADJUSTMENT_RANGE;
+//  int drate1=  (mi.rateoffset0&15)-ADJUSTMENT_RANGE;
+//  prm[ctx].dr[0][0] = drate0;
+//  prm[ctx].dr[0][1] = drate1;
+
+  return ((dr0 + ADJUSTMENT_RANGE) << 4) | ((dr1 + ADJUSTMENT_RANGE) & 15);
+}
+
+inline std::tuple<int, int> coding2drate( int code )
+{
+  return std::make_tuple( (code >> 4) - ADJUSTMENT_RANGE, (code & 15) - ADJUSTMENT_RANGE );
+}
+
+inline int probaInitConstant2coding( int p0, int p1 )
+{ 
+// get InitiId assuming slope=0
+//  int slope = (initId >> 3) - 4;
+//  int offset = ((initId & 7) * 18) + 1;
+//  int inistate = ((slope   * (qp - 16)) >> 1) + offset;
+//  int stateClip = inistate < 1 ? 1 : inistate > 127 ? 127 : inistate;
+
+  int offset = (128 * (p0 + p1)) / ((1 << PROB_BITS) + 1);
+  offset -= 1;
+
+  if( offset < 0 )
+  {
+    offset = 0;
+  }
+  offset /= 18;
+
+  if( offset > 7 )
+  {
+    offset = 7;
+  }
+  return (4 << 3) | offset;
+}
+

source/App/CabacTraining/io.h

+#pragma once
+#include <iostream>
+#include <vector>
+#include "Contexts.h"
+#include "data.h"
+using namespace std;
+
+
+extern int verbose;
+
+#if JVET_AH0176_LOW_DELAY_B_CTX
+static Models readModelParameters(std::istream &in)
+{
+  constexpr int nb_model_to_read = kNbModels;
+  Models prms;
+  assert(nb_model_to_read <= (int)prms.size());
+  for (int k = 0; k < nb_model_to_read; ++k)
+  {
+    in >> prms[k].initId;
+  }
+  for (int k = 0; k < nb_model_to_read; ++k)
+  {
+    in >> prms[k].log2windowsize;
+  }
+  for (int k = 0; k < nb_model_to_read; ++k)
+  {
+    in >> prms[k].adaptweight;
+  }
+  in >> prms[0].rateoffset[0] >> prms[0].rateoffset[1];
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+  in >> prms[1].rateoffset[0] >> prms[1].rateoffset[1];
+  in >> prms[2].rateoffset[0] >> prms[2].rateoffset[1];
+  in >> prms[3].rateoffset[0] >> prms[3].rateoffset[1];
+#else
+  for (int k = 1; k < 3; ++k)
+  {
+    prms[k].rateoffset[0] = prms[0].rateoffset[0];
+    prms[k].rateoffset[1] = prms[0].rateoffset[1];
+  }
+#endif
+  in.ignore(1024, '\n');
+  return prms;
+}
+#else
+static std::array<ModelParameters, 3> readModelParameters( std::istream &in )
+{
+  std::array<ModelParameters, 3> prms;
+  for( int k = 0; k < 3; ++k )
+  {
+    in >> prms[k].initId;
+  }
+  for( int k = 0; k < 3; ++k )
+  {
+    in >> prms[k].log2windowsize;
+  }
+  for( int k = 0; k < 3; ++k )
+  {
+    in >> prms[k].adaptweight;
+  }
+  in >> prms[0].rateoffset[0] >> prms[0].rateoffset[1];
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+  in >> prms[1].rateoffset[0] >> prms[1].rateoffset[1];
+  in >> prms[2].rateoffset[0] >> prms[2].rateoffset[1];
+#else
+  for( int k = 1; k < 3; ++k )
+  {
+    prms[k].rateoffset[0] = prms[0].rateoffset[0];
+    prms[k].rateoffset[1] = prms[0].rateoffset[1];
+  }
+#endif
+  in.ignore( 1024, '\n' );
+  return prms;
+}
+#endif
+
+static DataDb addDataFrame( istream &in, DataDb &db )
+{
+  char c = ( char ) in.peek();
+
+  while( in&&c == 'c' )
+  { 
+    // loop seq
+    std::string ctx;
+    in >> ctx;
+    if( ctx != "ctx" )
+    {
+      std::cerr << "[ERROR] invalid cabac file" << std::endl;
+      exit( -1 );
+    }
+    int ctxidx;
+    in >> ctxidx;
+
+    if( db.ctxidx == -1 )
+    {
+      db.ctxidx = ctxidx;
+      in.ignore( 1024, '\n' );
+      db.modelsBPI = readModelParameters( in );
+    }
+    else if( ctxidx != db.ctxidx )
+    {
+      std::cerr << "[ERROR] mixinf different ctx idx" << std::endl;
+      exit( -1 );
+    }
+    else
+    {
+      in.ignore( 1024, '\n' );
+      readModelParameters( in );
+    }
+
+    DataSequence v;
+    //in.ignore(1024, '\n');
+    c = ( char ) in.peek();
+
+    while( c == '#' )
+    {
+      in >> c;
+      std::string ss;
+      in >> ss;
+      if( ss == "SIZE" )
+      {
+        in >> v.filesize;
+      }
+      in.ignore( 1024, '\n' );
+      c = ( char ) in.peek();
+    }
+
+    DataFrame d;
+    char slicetype;
+    char reportslice;
+    while( c != 'c' && in >> d.poc >> slicetype >> d.qp >> d.switchBp >> d.tempCABAC >> reportslice >> d.p0 >> d.p1 >> d.rate >> d.weight
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+            >> d.drate0>>d.drate1
+#endif
+           )
+    { 
+      // loop frames
+      switch( slicetype )
+      {
+      case 'B':
+        d.type = B_SLICE;
+        break;
+      case 'P':
+        d.type = P_SLICE;
+        break;
+      case 'I':
+        d.type = I_SLICE;
+        break;
+#if JVET_AH0176_LOW_DELAY_B_CTX
+      case'L':
+        d.type = L_SLICE;
+        break;
+#endif
+      default:
+        std::cerr << "[ERROR] unkwon slice type" << std::endl;
+        exit( -1 );
+      }
+
+      switch( reportslice )
+      {
+      case 'B':
+        d.reportslice = B_SLICE;
+        break;
+      case 'P':
+        d.reportslice = P_SLICE;
+        break;
+      case 'I':
+        d.reportslice = I_SLICE;
+        break;
+#if JVET_AH0176_LOW_DELAY_B_CTX
+      case'L':
+        d.reportslice = L_SLICE;
+        break;
+#endif
+      default:
+        std::cerr << "[ERROR] unkwon slice type" << std::endl;
+        exit( -1 );
+      }
+      c = ( char ) in.get(); // get space
+      c = ( char ) in.peek();
+
+      if( c != '\n' )
+      {
+        std::string s;
+        in >> s;
+        d.bins.resize( s.size() );
+        for( int k = 0; k < ( int ) s.size(); ++k )
+        {
+          d.bins[k] = (s[k] == '1');
+        }
+        v.v.push_back( std::move( d ) );
+      }
+      in.ignore( 1024, '\n' );
+      c = ( char ) in.peek();
+    }
+
+    if( !v.v.empty() )
+    {
+      db.v.push_back( std::move( v ) );
+    }
+  }
+  return db;
+}
+
+static DataDb loadDataFrame(std::ifstream& file)
+{
+  DataDb db;
+  return addDataFrame( file, db );
+}
+
+static DataDb loadDataFrame()
+{
+  DataDb db;
+  return addDataFrame( std::cin, db );
+}
+
+#if JVET_AH0176_LOW_DELAY_B_CTX
+static void print(const Models &m)
+{
+  auto printField = [](const Models &m, int ModelParameters::* pi) 
+  {
+    for (int i = 0; i < (int)m.size(); ++i) 
+    {
+      std::cout << m[i].*pi << '\n';
+    }
+  };
+  printField(m, &ModelParameters::initId);
+  printField(m, &ModelParameters::log2windowsize);
+  printField(m, &ModelParameters::adaptweight);
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+  const int nbRateOffset = (int)m.size();
+#else
+  const int nbRateOffset = 1;
+#endif
+  for (int i = 0; i < nbRateOffset; ++i) 
+  {
+    std::cout << m[i].rateoffset[0] << '\n';
+    std::cout << m[i].rateoffset[1] << '\n';
+  }
+}
+#else
+static void print(const std::array<ModelParameters,3> &m)
+{
+  std::cout << m[0].initId << ",\n"
+            << m[1].initId << ",\n"
+            << m[2].initId << ",\n"
+            << m[0].log2windowsize << ",\n"
+            << m[1].log2windowsize << ",\n"
+            << m[2].log2windowsize << ",\n"
+            << m[0].adaptweight << ",\n"
+            << m[1].adaptweight << ",\n"
+            << m[2].adaptweight << ",\n"
+            << m[0].rateoffset[0] << ",\n"
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+            << m[0].rateoffset[1] << ",\n"
+            << m[1].rateoffset[0] << ",\n"
+            << m[1].rateoffset[1] << ",\n"
+            << m[2].rateoffset[0] << ",\n"
+            << m[2].rateoffset[1] << "\n";
+#else
+            << m[0].rateoffset[1] << "\n";
+#endif
+}
+#endif

source/App/CabacTraining/main.cpp

+#include <iomanip>
+#include <iostream>
+#include <fstream>
+#include <string>
+
+#include "program_options_lite.h"
+#include "Contexts.h"
+#include "io.h"
+#include "optimization.h"
+#include "simulator.h"
+
+using namespace std;
+namespace po = df::program_options_lite;
+static constexpr int CNU = 35;
+
+int main( int argc, char **argv )
+{
+  // parsing
+  bool        doHelp = false;
+  bool        keepModel = false;
+  std::string outfile;
+  std::string inputFile = "";
+#if JVET_AH0176_LOW_DELAY_B_CTX
+  std::string sliceToTrain;
+#endif
+  po::Options opts;
+  // clang-format off
+  opts.addOptions()
+    ( "help", doHelp, false, "optimize cabac contexts. Usage:\ncat *.cabac_xxx | cabacTrain --output fileout.txt" )
+    ( "keepModel", keepModel, false, "if no bin, keep old parameters instead of putting default parameters" )
+#if JVET_AH0176_LOW_DELAY_B_CTX
+    ("sliceToTrain", sliceToTrain, std::string("all"), "all or B/P/I/L")
+#endif
+    ( "output", outfile, std::string( "cabac_prm.txt" ), "filename of the resulting parameters" )
+    ( "input", inputFile, std::string( "" ), "filename of the input bins" )
+    ;
+  // clang-format on
+  po::setDefaults( opts );
+  po::ErrorReporter         err;
+  const list<const char *> &argvUnhandled = po::scanArgv( opts, argc, (const char **)argv, err );
+
+  for( list<const char *>::const_iterator it = argvUnhandled.begin(); it != argvUnhandled.end(); it++ )
+  {
+    msg( ERROR, "Unhandled argument ignored: `%s'\n", *it );
+  }
+
+  if( doHelp )
+  {
+    po::doHelp( cout, opts );
+    return -1;
+  }
+
+  if( err.is_errored )
+  {
+    return -1;
+  }
+
+#if JVET_AH0176_LOW_DELAY_B_CTX
+  SliceType toTrain = NUMBER_OF_SLICE_TYPES;
+
+  if (sliceToTrain == "B") 
+  {
+    toTrain = B_SLICE;
+  }
+  else if (sliceToTrain == "P")
+  {
+    toTrain = P_SLICE;
+  }
+  else if (sliceToTrain == "I") 
+  {
+    toTrain = I_SLICE;
+  }
+  else if (sliceToTrain == "all") 
+  {
+    toTrain = NUMBER_OF_SLICE_TYPES;
+  }
+  else if (sliceToTrain == "L") 
+  {
+    toTrain = L_SLICE;
+  }
+  else 
+  {
+    std::cerr << "[ERROR] unknown slice to train" << endl;
+  }
+#endif
+
+  DataDb db;
+
+  if( inputFile != "" )
+  {
+    ifstream in( inputFile.c_str() );
+    db = loadDataFrame( in );
+  }
+  else
+  {
+    db = loadDataFrame();
+  }
+
+#if JVET_AH0176_LOW_DELAY_B_CTX
+  std::vector<SliceType> toOpt = { B_SLICE, P_SLICE, I_SLICE, L_SLICE };
+  std::array<bool, kNbModels>    sliceActivation = {};
+#else
+  std::vector<SliceType> toOpt = { B_SLICE, P_SLICE, I_SLICE };
+  std::array<bool, 3>    sliceActivation = {};
+#endif
+  if( db.ctxidx < 0 )
+  {
+    std::cout << "[ERROR] no context to train found" << std::endl;
+    return -1;
+  }
+  std::cout << "Ctx " << db.ctxidx << std::endl;
+  bool hasBin = false;
+  for( int k = 0; k < (int)toOpt.size(); ++k )
+  {
+    SliceType st = toOpt[k];
+    sliceActivation[k] = hasBin1Db( db.v, st );
+    hasBin = true;
+  }
+
+#if JVET_AH0176_LOW_DELAY_B_CTX
+  Models modelsgreedy = db.modelsBPI;
+#else
+  std::array<ModelParameters, 3> modelsgreedy = db.modelsBPI;
+#endif
+  std::cout << "Before:\n";
+  print( db.modelsBPI );
+  std::cout << std::endl;
+  if( hasBin )
+  {
+    for( int k = 0; k < (int)toOpt.size(); ++k )
+    {
+      SliceType st = toOpt[k];
+      if( sliceActivation[k] )
+      {
+#if JVET_AH0176_LOW_DELAY_B_CTX
+        if (toTrain == NUMBER_OF_SLICE_TYPES || k == toTrain)
+        {
+#endif
+        std::cout << "Optimized parameter slice type " << k << std::endl;
+        modelsgreedy[(int)st] = getBestGreedy( db, st );
+        db.modelsBPI[(int)st] = modelsgreedy[(int)st];
+
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+        std::cout << "Optimized rate offsets " << std::endl;
+        auto drate = getBestGreedyDrate( db, st );
+
+        db.modelsBPI[k].rateoffset[0] = drate[0];
+        db.modelsBPI[k].rateoffset[1] = drate[1];
+        modelsgreedy[k].rateoffset[0] = drate[0];
+        modelsgreedy[k].rateoffset[1] = drate[1];
+#endif
+#if JVET_AH0176_LOW_DELAY_B_CTX
+        }
+#endif
+      }
+      else
+      {
+        if( keepModel )
+        {
+          // do not change the prm
+        }
+        else
+        {
+          std::cout << "Default model (no bin) for slice type " << k << std::endl;
+          modelsgreedy[k].initId = db.modelsBPI[(int)st].initId = CNU;
+          modelsgreedy[k].log2windowsize = db.modelsBPI[(int)st].log2windowsize = DWS;
+          modelsgreedy[k].adaptweight = db.modelsBPI[(int)st].adaptweight = DWE;
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+          modelsgreedy[k].rateoffset[0] = DWO;
+          modelsgreedy[k].rateoffset[1] = DWO;
+#endif
+        }
+      }
+    }
+
+#if !JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+    std::cout << "Optimized rate offsets " << std::endl;
+    auto drate = getBestGreedyDrate( db, sliceActivation );
+    for( int k = 0; k < 3; ++k )
+    {
+      db.modelsBPI[k].rateoffset[0] = drate[0];
+      db.modelsBPI[k].rateoffset[1] = drate[1];
+      modelsgreedy[k].rateoffset[0] = drate[0];
+      modelsgreedy[k].rateoffset[1] = drate[1];
+    }
+#endif
+  }
+  else
+  {
+    if( keepModel )
+    {
+      std::cout << "Skipped (no bin)" << std::endl;
+    }
+    else
+    {
+      std::cout << "Default model (no bin)" << std::endl;
+#if JVET_AH0176_LOW_DELAY_B_CTX
+      for (int k = 0; k < kNbModels; ++k)
+#else
+      for( int k = 0; k < 3; ++k )
+#endif
+      {
+        modelsgreedy[k].initId = CNU;
+        modelsgreedy[k].log2windowsize = DWS;
+        modelsgreedy[k].adaptweight = DWE;
+        modelsgreedy[k].rateoffset[0] = DWO;
+        modelsgreedy[k].rateoffset[1] = DWO;
+      }
+    }
+  }
+  std::cout << "\nAfter:\n";
+  print( modelsgreedy );
+
+  if( !outfile.empty() )
+  {
+    ofstream file( outfile, ios::binary );
+#if JVET_AH0176_LOW_DELAY_B_CTX
+    file << db.ctxidx << " A ";
+    for (int i = 0; i < kNbModels; ++i)
+    {
+      file << modelsgreedy[i].initId << ' ';
+    }
+    for (int i = 0; i < kNbModels; ++i) 
+    {
+      file << modelsgreedy[i].log2windowsize << ' ';
+    }
+    for (int i = 0; i < kNbModels; ++i)
+    {
+      file << modelsgreedy[i].adaptweight << ' ';
+    }
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+    const int nbRateOffset = kNbModels;
+#else
+    const int nbRateOffset = 1;
+#endif
+    for (int i = 0; i < nbRateOffset; ++i) 
+    {
+      file << modelsgreedy[i].rateoffset[0] << ' ' << modelsgreedy[i].rateoffset[1] << ' ';
+    }
+    file << std::endl;
+#else
+    file << db.ctxidx << " A "
+         << modelsgreedy[0].initId << ' '         << modelsgreedy[1].initId << ' '         << modelsgreedy[2].initId << ' '
+         << modelsgreedy[0].log2windowsize << ' ' << modelsgreedy[1].log2windowsize << ' ' << modelsgreedy[2].log2windowsize << ' '
+         << modelsgreedy[0].adaptweight << ' '    << modelsgreedy[1].adaptweight << ' '    << modelsgreedy[2].adaptweight << ' '
+         << modelsgreedy[0].rateoffset[0] << ' ' << modelsgreedy[0].rateoffset[1]
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+         << ' ' << modelsgreedy[1].rateoffset[0] << ' ' << modelsgreedy[1].rateoffset[1]
+         << ' ' << modelsgreedy[2].rateoffset[0] << ' ' << modelsgreedy[2].rateoffset[1]
+#endif
+      << std::endl;
+#endif
+  }
+}

source/App/CabacTraining/optimization.h

+#pragma once
+#include "simulator.h"
+
+static constexpr int log2winsize[13] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13 };
+static constexpr int adaptweight[5] = { 4, 11, 18, 25, 32 };
+
+// get best model for each ctx given a set of frames
+inline ModelParameters getBestGreedy( const DataDb &db, SliceType st )
+{
+  constexpr int slopeStart = 0;
+  constexpr int slopeEnd = 7;
+  constexpr int offsetstart = 0;
+  constexpr int offsetend = 7;
+  constexpr int winSizesIdxStart = 0;
+  constexpr int winSizesIdxEnd = 12;
+  constexpr int weightIdxStart = 0;
+  constexpr int weightIdxEnd = 4;
+  ModelParameters prm = db.modelsBPI[st];
+  ModelParameters bestprm = prm;
+  double bestc = std::numeric_limits<double>::max();
+
+  for( int slope = slopeStart; slope <= slopeEnd; ++slope )
+  {
+    for( int offset = offsetstart; offset <= offsetend; ++offset )
+    {
+      for( int winSizesIdx = winSizesIdxStart; winSizesIdx <= winSizesIdxEnd; ++winSizesIdx )
+      {
+        for( int weightIdx = weightIdxStart; weightIdx <= weightIdxEnd; ++weightIdx )
+        {
+          prm.initId = (slope << 3) | (offset);
+          prm.log2windowsize = log2winsize[winSizesIdx];
+          prm.adaptweight = adaptweight[weightIdx];
+          double c = cost1Db( db.v, prm, st );
+
+          if( c < bestc )
+          {
+            bestc = c;
+            bestprm = prm;
+          }
+        }
+      }
+    }
+  }
+  return bestprm;
+}
+
+// get best model optimizing drate
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+inline std::array<int, 2> getBestGreedyDrate( const DataDb &db, const SliceType st )
+#else
+inline std::array<int,2> getBestGreedyDrate(const DataDb &db,const std::array<bool,3> &sliceActivation)
+#endif
+{
+  constexpr int drateStartIdx = 0;
+  constexpr int drateEndIdx = 15;
+  constexpr int kMaxRate = 12; // theoritcal is PROB_BITS=15
+  int bestdrate[4] = { 7,7,7,7 };
+  double bestc = std::numeric_limits<double>::max();
+
+  for( int drate00 = drateStartIdx; drate00 <= drateEndIdx; drate00++ )
+  {
+    for( int drate01 = drateStartIdx; drate01 <= drateEndIdx; drate01++ )
+    {
+      for( int drate10 = drateStartIdx; drate10 <= drateEndIdx; drate10++ )
+      {
+        for( int drate11 = drateStartIdx; drate11 <= drateEndIdx; drate11++ )
+        {
+          // check if ok for one slice type at least
+          bool notOk = true;
+#if !JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+          for( int st = 0; st < 3; ++st )
+          {
+            if( !sliceActivation[st] )
+            {
+              continue;
+            }
+#endif
+
+            ModelParameters prm = db.modelsBPI[st];
+            int rate0 = 2 + ((prm.log2windowsize >> 2) & 3);
+            int rate1 = 3 + rate0 + (prm.log2windowsize & 3);
+            // avoid testing redundant prms
+            int rateUsed0 = rate0 + drate00 - ADJUSTMENT_RANGE;
+            int rateUsed1 = rate1 + drate01 - ADJUSTMENT_RANGE;
+
+            if( rateUsed0 < 2 || rateUsed1 < 2 || rateUsed0 > kMaxRate || rateUsed1 > kMaxRate )
+            {
+              continue;
+            }
+
+            rateUsed0 = rate0 + drate10 - ADJUSTMENT_RANGE;
+            rateUsed1 = rate1 + drate11 - ADJUSTMENT_RANGE;
+
+            if( rateUsed0 < 2 || rateUsed1 < 2 || rateUsed0 > kMaxRate || rateUsed1 > kMaxRate )
+            {
+              continue;
+            }
+            notOk = false;
+#if !JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+          }
+#endif
+
+          if( notOk )
+          {
+            continue;
+          }
+
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+          auto prms = db.modelsBPI[st];
+
+          prms.rateoffset[0] = ( drate00 << 4 ) | ( drate01 );
+          prms.rateoffset[1] = ( drate10 << 4 ) | ( drate11 );
+#else
+          auto prms = db.modelsBPI;
+
+          for( int st = 0; st < 3; ++st )
+          {
+            prms[st].rateoffset[0] = (drate00 << 4) | (drate01);
+            prms[st].rateoffset[1] = (drate10 << 4) | (drate11);
+          }
+#endif
+
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+          double c = cost1Db( db.v, prms, st );
+#else
+          double c = cost1Db( db.v, prms, sliceActivation );
+#endif
+
+          if( c < bestc )
+          {
+            bestc = c;
+            bestdrate[0] = drate00;
+            bestdrate[1] = drate01;
+            bestdrate[2] = drate10;
+            bestdrate[3] = drate11;
+          }
+        }
+      }
+    }
+  }
+
+  std::array<int, 2> bestDrate;
+  bestDrate[0] = (bestdrate[0] << 4) | bestdrate[1];
+  bestDrate[1] = (bestdrate[2] << 4) | bestdrate[3];
+  return bestDrate;
+}

source/App/CabacTraining/simulator.h

+#pragma once
+#include <iostream>
+#include <vector>
+#include <tuple>
+#include "Contexts.h"
+#include "data.h"
+using namespace std;
+
+
+inline void initDefaultParameters( const ModelParameters &prms, const DataFrame &slice, BinProbModel_Std &cabac )
+{
+  // use frame extracted parameters (should be the same as default one, except for proba)
+  cabac.setState( { slice.p0,slice.p1 } );
+  cabac.setWinSizes( slice.rate );
+  cabac.setAdaptRateWeight( slice.weight );
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+  cabac.setAdaptRateOffset( slice.drate0, 0 );
+  cabac.setAdaptRateOffset( slice.drate1, 1 );
+#else
+  cabac.setAdaptRateOffset( prms.rateoffset[0], 0 );
+  cabac.setAdaptRateOffset( prms.rateoffset[1], 1 );
+#endif
+}
+
+inline std::pair<uint16_t, uint16_t> coding2ProbaInit( int initId, int qp )
+{
+  // getP0
+  int slope = (initId >> 3) - 4;
+  int offset = ((initId & 7) * 18) + 1;
+  int inistate = ((slope * (qp - 16)) >> 1) + offset;
+  int stateClip = (inistate < 1) ? 1 : (inistate > 127 ? 127 : inistate);
+  uint16_t p0 = (stateClip << 8)& MASK_0;
+  uint16_t p1 = (stateClip << 8)& MASK_1;
+
+  return std::make_pair( p0, p1 );
+}
+
+inline void initNewParameters( const ModelParameters &prms, const DataFrame &slice, BinProbModel_Std &cabac )
+{
+  if( !slice.tempCABAC )
+  {
+    // use the proba model
+    cabac.setState( coding2ProbaInit( prms.initId, slice.qp ) );
+  }
+
+  cabac.setLog2WindowSize( prms.log2windowsize );
+  cabac.setAdaptRateWeight( prms.adaptweight );
+#if JVET_AG0196_WINDOWS_OFFSETS_SLICETYPE
+  cabac.setAdaptRateOffset( prms.rateoffset[0], 0 );
+  cabac.setAdaptRateOffset( prms.rateoffset[1], 1 );
+#endif
+}
+
+inline uint64_t cost1Frame( const DataFrame &slice, BinProbModel_Std &cabac )
+{
+  uint64_t cost{};
+  for( auto b : slice.bins )
+  {
+    cabac.estFracBitsUpdate( b, cost );
+  }
+  return cost;
+}
+
+inline uint64_t cost1Sequence( const DataSequence &seq, const ModelParameters &prms, SliceType st )
+{
+  BinProbModel_Std cabac;
+  uint64_t cost = 0;
+
+  for( const auto &slice : seq.v )
+  {
+    if( slice.reportslice == st )
+    {
+      initDefaultParameters( prms, slice, cabac );
+      initNewParameters( prms, slice, cabac );
+      cost += cost1Frame( slice, cabac );
+    }
+  }
+  return cost;
+}
+
+inline bool hasBin1Frame( const DataFrame &slice )
+{
+  return !slice.bins.empty();
+}
+
+inline bool hasBin1Sequence( const DataSequence &seq, SliceType st )
+{
+  for( const auto &slice : seq.v )
+  {
+    if( slice.reportslice == st )
+    {
+      if( hasBin1Frame( slice ) )
+      {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+inline bool hasBin1Db( const DataSequences &seqs, SliceType st )
+{
+  for( const auto &seq : seqs )
+  {
+    if( hasBin1Sequence( seq, st ) )
+    {
+      return true;
+    }
+  }
+  return false;
+}
+
+inline double cost1Db( const DataSequences &seqs, const ModelParameters &prms, SliceType st )
+{
+  double costr = 0.;
+  for( const auto &seq : seqs )
+  {
+    auto c = cost1Sequence( seq, prms, st );
+    costr += ( double ) c / (8 * seq.filesize);
+  }
+  return costr;
+}
+#if JVET_AH0176_LOW_DELAY_B_CTX
+inline double cost1Db( const DataSequences &seqs, const std::array<ModelParameters, kNbModels> &prms, const std::array<bool, kNbModels> &sliceActivation )
+#else
+inline double cost1Db( const DataSequences &seqs, const std::array<ModelParameters, 3> &prms, const std::array<bool, 3> &sliceActivation )
+#endif
+{
+  double costr = 1;
+  for( const auto &seq : seqs )
+  {
+    double costr2 = 0.;
+
+#if JVET_AH0176_LOW_DELAY_B_CTX
+    for( int i = 0; i < kNbModels; ++i )
+#else
+    for( int i = 0; i < 3; ++i )
+#endif
+    {
+      if( sliceActivation[i] )
+      {
+        costr2 += cost1Sequence( seq, prms[i], ( SliceType ) i );
+      }
+    }
+    costr += costr2 / (8 * seq.filesize);
+  }
+  return costr;
+}

source/App/DecoderAnalyserApp/CMakeLists.txt

@@ -27,7 +27,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 target_compile_definitions( ${EXE_NAME} PUBLIC RExt__DECODER_DEBUG_BIT_STATISTICS=1 )
 target_compile_definitions( ${EXE_NAME} PUBLIC RExt__DECODER_DEBUG_TOOL_STATISTICS=1 )
 
-if( SET_ENABLE_TRACING )
+if( DEFINED ENABLE_TRACING )
   if( ENABLE_TRACING )
     target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_TRACING=1 )
   else()
@@ -35,24 +35,12 @@ if( SET_ENABLE_TRACING )
   endif()
 endif()
 
-if( OpenMP_FOUND )
-  if( SET_ENABLE_SPLIT_PARALLELISM )
-    if( ENABLE_SPLIT_PARALLELISM )
-      target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_SPLIT_PARALLELISM=1 )
-    else()
-      target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_SPLIT_PARALLELISM=0 )
-    endif()
-  endif()
-  if( SET_ENABLE_WPP_PARALLELISM )
-    if( ENABLE_WPP_PARALLELISM )
-      target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_WPP_PARALLELISM=1 )
-    else()
-      target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_WPP_PARALLELISM=0 )
-    endif()
+if( DEFINED ENABLE_HIGH_BITDEPTH )
+  if( ENABLE_HIGH_BITDEPTH )
+    target_compile_definitions( ${EXE_NAME} PUBLIC RExt__HIGH_BIT_DEPTH_SUPPORT=1 )
+  else()
+    target_compile_definitions( ${EXE_NAME} PUBLIC RExt__HIGH_BIT_DEPTH_SUPPORT=0 )
   endif()
-else()
-  target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_SPLIT_PARALLELISM=0 )
-  target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_WPP_PARALLELISM=0 )
 endif()
 
 if( CMAKE_COMPILER_IS_GNUCC AND BUILD_STATIC )
@@ -60,7 +48,7 @@ if( CMAKE_COMPILER_IS_GNUCC AND BUILD_STATIC )
   target_compile_definitions( ${EXE_NAME} PUBLIC ENABLE_WPP_STATIC_LINK=1 )
 endif()
 
-target_link_libraries( ${EXE_NAME} CommonAnalyserLib DecoderAnalyserLib Utilities Threads::Threads ${ADDITIONAL_LIBS} )
+target_link_libraries( ${EXE_NAME} CommonAnalyserLib DecoderAnalyserLib Utilities ${ADDITIONAL_LIBS} )
 
 # lldb custom data formatters
 if( XCODE )