From 78891c9c90c66183b5d48b4ca46da13bfb7556a9 Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Thu, 3 Aug 2023 11:36:06 +0200 Subject: [PATCH 01/12] unified scripts API --- training/tools/convert_dataset.py | 3 +- training/tools/create_config.py | 46 ++ .../NN_Filtering_HOP/config.json | 396 ----------- .../NN_Filtering_HOP/default_config.json | 670 ++++++++++++++++++ .../NN_Filtering_HOP/paths.json | 192 +++++ .../NN_Filtering_HOP/training/cfg/debug.json | 22 - .../NN_Filtering_HOP/training/cfg/stage1.json | 17 - .../training/cfg/stage1_fast.json | 17 - .../NN_Filtering_HOP/training/cfg/stage2.json | 18 - .../training/cfg/stage2_fast.json | 17 - .../NN_Filtering_HOP/training/cfg/stage3.json | 20 - .../training/cfg/stage3_fast.json | 17 - .../training/cfg/training_default.json | 85 --- .../NN_Filtering_HOP/training/main.py | 46 +- .../NN_Filtering_HOP/training/trainer.py | 4 +- 15 files changed, 936 insertions(+), 634 deletions(-) create mode 100644 training/tools/create_config.py delete mode 100644 training/training_scripts/NN_Filtering_HOP/config.json create mode 100644 training/training_scripts/NN_Filtering_HOP/default_config.json create mode 100644 training/training_scripts/NN_Filtering_HOP/paths.json delete mode 100644 training/training_scripts/NN_Filtering_HOP/training/cfg/debug.json delete mode 100644 training/training_scripts/NN_Filtering_HOP/training/cfg/stage1.json delete mode 100644 training/training_scripts/NN_Filtering_HOP/training/cfg/stage1_fast.json delete mode 100644 training/training_scripts/NN_Filtering_HOP/training/cfg/stage2.json delete mode 100644 training/training_scripts/NN_Filtering_HOP/training/cfg/stage2_fast.json delete mode 100644 training/training_scripts/NN_Filtering_HOP/training/cfg/stage3.json delete mode 100644 training/training_scripts/NN_Filtering_HOP/training/cfg/stage3_fast.json delete mode 100644 training/training_scripts/NN_Filtering_HOP/training/cfg/training_default.json diff --git a/training/tools/convert_dataset.py b/training/tools/convert_dataset.py index dfa6ef35e..e4bb6f333 100644 --- a/training/tools/convert_dataset.py +++ b/training/tools/convert_dataset.py @@ -109,8 +109,7 @@ parser.add_argument( "--json_config", action="store", type=str, - help="global configuration file", - required=True, + help="global configuration file" ) parser.add_argument( diff --git a/training/tools/create_config.py b/training/tools/create_config.py new file mode 100644 index 000000000..ce7c6bfbc --- /dev/null +++ b/training/tools/create_config.py @@ -0,0 +1,46 @@ +""" +/* The copyright in this software is being made available under the BSD +* License, included below. This software may be subject to other third party +* and contributor rights, including patent rights, and no such rights are +* granted under this license. +* +* Copyright (c) 2010-2023, ITU/ISO/IEC +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are met: +* +* * Redistributions of source code must retain the above copyright notice, +* this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright notice, +* this list of conditions and the following disclaimer in the documentation +* and/or other materials provided with the distribution. +* * Neither the name of the ITU/ISO/IEC nor the names of its contributors may +* be used to endorse or promote products derived from this software without +* specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS +* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +* THE POSSIBILITY OF SUCH DAMAGE. +""" + +import json +import sys +import jsonmerge + +# CLI: python create_config.py default_config.json path.json other_custom.json... +config = {} +for config_filename in sys.argv[1:]: + with open(config_filename, "r") as file: + j = json.load(file) + config = jsonmerge.merge(config, j) + +print(json.dumps(config, indent=1)) diff --git a/training/training_scripts/NN_Filtering_HOP/config.json b/training/training_scripts/NN_Filtering_HOP/config.json deleted file mode 100644 index 04540633c..000000000 --- a/training/training_scripts/NN_Filtering_HOP/config.json +++ /dev/null @@ -1,396 +0,0 @@ -{ - "binaries" : { "ffmpeg" : "ffmpeg", - "sadl_path": "/path/to/src/sadl" - }, - "dataset" : { - "div2k_train" : { - "//" : "dataset of png to convert", - "start" : 1, - "count" : 800, - "path" : "/path/to/DIV2K/DIV2K_train_HR" - }, - "div2k_valid" : { - "start" : 801, - "count" : 100, - "path" : "/path/to/DIV2K/DIV2K_valid_HR" - }, - "bvi" : { - "//" : "dataset of yuv", - "path" : "/path/to/bviorg", - "dataset_file" : "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/bvi.json" - }, - "bvi_valid" : { - "//" : "dataset of yuv", - "path" : "/path/to/bviorg", - "dataset_file" : "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/bvi_valid.json" - }, - "tvd" : { - "//" : "dataset of yuv", - "path" : "/path/to/tvdorg", - "dataset_file" : "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/tvd.json" - }, - "tvd_valid" : { - "//" : "dataset of yuv", - "path" : "/path/to/tvdorg", - "dataset_file" : "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/tvd_valid.json" - } - }, - "stage1" : { - "yuv" : { - "//" : "path to store yuv files dataset", - "path" : "/path/to/stage1/yuv", - "dataset_file" : "dataset.json" - }, - "yuv_valid" : { - "//" : "path to store yuv files dataset", - "path" : "/path/to/stage1/yuv", - "dataset_file" : "dataset_valid.json" - }, - "encdec" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage1/encdec", - "dump_dir" : "dump", - "enc_dir" : "enc", - "dec_dir" : "dec", - "cfg_dir" : "cfg", - "qps" : "19 22 24 27 29 32 34 37 39 42", - "vtm_enc" : "/path/to/src5.0/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.0/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.0/cfg/encoder_intra_vtm.cfg", - "dataset_file" : "dataset.json", - "encode_script" : "encode_decode_dataset.sh" - }, - "encdec_valid" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage1/encdec", - "dump_dir" : "dump_valid", - "enc_dir" : "enc_valid", - "dec_dir" : "dec_valid", - "cfg_dir" : "cfg_valid", - "qps" : "19 22 24 27 29 32 34 37 39 42", - "vtm_enc" : "/path/to/src5.0/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.0/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.0/cfg/encoder_intra_vtm.cfg", - "dataset_file" : "dataset_valid.json", - "encode_script" : "encode_decode_dataset_valid.sh" - }, - "dataset" : { - "//" : "path to store the full dataset which will be used by the training", - "path" : "/path/to/stage1/dataset", - "dataset_file" : "div2k.json" - }, - "dataset_valid" : { - "//" : "path to store the full dataset which will be used by the training", - "path" : "/path/to/stage1/dataset", - "dataset_file" : "div2k_valid.json" - }, - "training" : { - "//" : "out_filename: set to null for stdout. all path relative to save_base_dir", - "path" : "/path/to/stage1/train", - "ckpt_dir" : "ckpt", - "ckpt_reload" : "last.ckpt", - "model_onnx_filename" : "last.onnx", - "loggers" : { - "PrintLogger" : { "out_filename" : "logs/log.txt" }, - "ProgressLogger" : { "out_filename" : null }, - "TensorboardLogger" : { "log_dir" : "tensorboard" } - } - }, - "conversion" : { - "//" : "path to output the model. input model is taken in training/ckpt_dir", - "path" : "/path/to/stage1/train", - "model_filename" : "model_float.sadl" - } - }, - "stage2" : { - "yuv_tvd" : { - "//" : "path to store yuv files dataset", - "path" : "/path/to/stage2/yuv", - "dataset_file" : "dataset_tvd.json" - }, - "yuv_tvd_valid" : { - "//" : "path to store yuv files dataset", - "path" : "/path/to/stage2/yuv", - "dataset_file" : "dataset_tvd_valid.json" - }, - "yuv_bvi" : { - "//" : "path to store yuv files dataset", - "path" : "/path/to/stage2/yuv", - "dataset_file" : "dataset_bvi.json" - }, - "yuv_bvi_valid" : { - "//" : "path to store yuv files dataset", - "path" : "/path/to/stage2/yuv", - "dataset_file" : "dataset_bvi_valid.json" - }, - "encdec_bvi" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage2/encdec", - "dump_dir" : "dump_bvi", - "enc_dir" : "enc_bvi", - "dec_dir" : "dec_bvi", - "cfg_dir" : "cfg_bvi", - "qps" : "22 27 32 37 42", - "vtm_enc" : "/path/to/src5.1/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.1/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.1/cfg/encoder_randomaccess_vtm.cfg", - "vtm_option" : "--NnlfHopDebugOption=1", - "vtm_model" : "stage1/conversion", - "dataset_file" : "dataset_bvi.json", - "encode_script" : "encode_decode_dataset_bvi.sh" - }, - "encdec_bvi_valid" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage2/encdec", - "dump_dir" : "dump_bvi_valid", - "enc_dir" : "enc_bvi_valid", - "dec_dir" : "dec_bvi_valid", - "cfg_dir" : "cfg_bvi_valid", - "qps" : "22 27 32 37 42", - "vtm_enc" : "/path/to/src5.1/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.1/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.1/cfg/encoder_randomaccess_vtm.cfg", - "vtm_option" : "--NnlfHopDebugOption=1", - "vtm_model" : "stage1/conversion", - "dataset_file" : "dataset_bvi_valid.json", - "encode_script" : "encode_decode_dataset_bvi_valid.sh" - }, - "encdec_tvd" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage2/encdec", - "dump_dir" : "dump_tvd", - "enc_dir" : "enc_tvd", - "dec_dir" : "dec_tvd", - "cfg_dir" : "cfg_tvd", - "qps" : "22 27 32 37 42", - "vtm_enc" : "/path/to/src5.1/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.1/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.1/cfg/encoder_randomaccess_vtm.cfg", - "vtm_option" : "--NnlfHopDebugOption=1", - "vtm_model" : "stage1/conversion", - "dataset_file" : "dataset_tvd.json", - "encode_script" : "encode_decode_dataset_tvd.sh" - }, - "encdec_tvd_valid" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage2/encdec", - "dump_dir" : "dump_tvd_valid", - "enc_dir" : "enc_tvd_valid", - "dec_dir" : "dec_tvd_valid", - "cfg_dir" : "cfg_tvd_valid", - "qps" : "22 27 32 37 42", - "vtm_enc" : "/path/to/src5.1/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.1/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.1/cfg/encoder_randomaccess_vtm.cfg", - "vtm_option" : "--NnlfHopDebugOption=1", - "vtm_model" : "stage1/conversion", - "dataset_file" : "dataset_tvd_valid.json", - "encode_script" : "encode_decode_dataset_tvd_valid.sh" - }, - "dataset" : { - "//" : "path to store the full dataset which will be used by the training", - "path" : "/path/to/stage2/dataset", - "dataset_files" : { - "div2k" : { - "dataset_file" : "div2k.json", - "augs" : [ ], - "aug_prob" : 0.0 - }, - "bvi" : { - "dataset_file" : "bvi.json", - "augs" : [ - "vflip", - "hflip", - "transpose" - ], - "aug_prob" : 0.2 - }, - "tvd" : { - "dataset_file" : "tvd.json", - "augs" : [ - "vflip", - "hflip", - "transpose" - ], - "aug_prob" : 0.2 - } - } - }, - "dataset_valid" : { - "//" : "path to store the full dataset which will be used by the training", - "path" : "/path/to/stage2/dataset", - "dataset_files" : { - "div2k" : { - "dataset_file" : "div2k_valid.json", - "augs" : [ ], - "aug_prob" : 0.0 - }, - "bvi" : { - "dataset_file" : "bvi_valid.json", - "augs" : [ ], - "aug_prob" : 0.2 - }, - "tvd" : { - "dataset_file" : "tvd_valid.json", - "augs" : [ ], - "aug_prob" : 0.2 - } - } - }, - "training" : { - "//" : "out_filename: set to null for stdout. all path relative to save_base_dir", - "path" : "/path/to/stage2/train", - "ckpt_dir" : "ckpt", - "ckpt_reload" : "last.ckpt", - "model_onnx_filename" : "last.onnx", - "loggers" : { - "PrintLogger" : { "out_filename" : "logs/log.txt" }, - "ProgressLogger" : { "out_filename" : null }, - "TensorboardLogger" : { "log_dir" : "tensorboard" } - } - }, - "conversion" : { - "//" : "path to output the model. input model is taken in training/ckpt_dir", - "path" : "/path/to/stage2/train", - "model_filename" : "model_float.sadl" - } - }, - "stage3" : { - "encdec_bvi" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage3/encdec", - "dump_dir" : "dump_bvi", - "enc_dir" : "enc_bvi", - "dec_dir" : "dec_bvi", - "cfg_dir" : "cfg_bvi", - "qps" : "22 27 32 37 42", - "vtm_enc" : "/path/to/src5.1/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.1/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.1/cfg/encoder_randomaccess_vtm.cfg", - "vtm_option" : "", - "vtm_model" : "stage2/conversion", - "dataset_file" : "dataset_bvi.json", - "encode_script" : "encode_decode_dataset_bvi.sh" - }, - "encdec_bvi_valid" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage3/encdec", - "dump_dir" : "dump_bvi_valid", - "enc_dir" : "enc_bvi_valid", - "dec_dir" : "dec_bvi_valid", - "cfg_dir" : "cfg_bvi_valid", - "qps" : "22 27 32 37 42", - "vtm_enc" : "/path/to/src5.1/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.1/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.1/cfg/encoder_randomaccess_vtm.cfg", - "vtm_option" : "", - "vtm_model" : "stage2/conversion", - "dataset_file" : "dataset_bvi_valid.json", - "encode_script" : "encode_decode_dataset_bvi_valid.sh" - }, - "encdec_tvd" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage3/encdec", - "dump_dir" : "dump_tvd", - "enc_dir" : "enc_tvd", - "dec_dir" : "dec_tvd", - "cfg_dir" : "cfg_tvd", - "qps" : "22 27 32 37 42", - "vtm_enc" : "/path/to/src5.1/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.1/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.1/cfg/encoder_randomaccess_vtm.cfg", - "vtm_option" : "", - "vtm_model" : "stage2/conversion", - "dataset_file" : "dataset_tvd.json", - "encode_script" : "encode_decode_dataset_tvd.sh" - }, - "encdec_tvd_valid" : { - "//" : "path to store the shell script and all the generated files by the encoder/decoder", - "path" : "/path/to/stage3/encdec", - "dump_dir" : "dump_tvd_valid", - "enc_dir" : "enc_tvd_valid", - "dec_dir" : "dec_tvd_valid", - "cfg_dir" : "cfg_tvd_valid", - "qps" : "22 27 32 37 42", - "vtm_enc" : "/path/to/src5.1/bin/EncoderAppStatic", - "vtm_dec" : "/path/to/src5.1/bin/DecoderAppStatic", - "vtm_cfg" : "/path/to/src5.1/cfg/encoder_randomaccess_vtm.cfg", - "vtm_option" : "", - "vtm_model" : "stage2/conversion", - "dataset_file" : "dataset_tvd_valid.json", - "encode_script" : "encode_decode_dataset_tvd_valid.sh" - }, - "dataset" : { - "//" : "path to store the full dataset which will be used by the training", - "path" : "/path/to/stage3/dataset", - "dataset_files" : { - "div2k" : { - "dataset_file" : "div2k.json", - "augs" : [ ], - "aug_prob" : 0.0 - }, - "bvi" : { - "dataset_file" : "bvi.json", - "augs" : [ - "vflip", - "hflip", - "transpose" - ], - "aug_prob" : 0.2 - }, - "tvd" : { - "dataset_file" : "tvd.json", - "augs" : [ - "vflip", - "hflip", - "transpose" - ], - "aug_prob" : 0.2 - } - } - }, - "dataset_valid" : { - "//" : "path to store the full dataset which will be used by the training", - "path" : "/path/to/stage3/dataset", - "dataset_files" : { - "div2k" : { - "dataset_file" : "div2k_valid.json", - "augs" : [ ], - "aug_prob" : 0.0 - }, - "bvi" : { - "dataset_file" : "bvi_valid.json", - "augs" : [ ], - "aug_prob" : 0.2 - }, - "tvd" : { - "dataset_file" : "tvd_valid.json", - "augs" : [ ], - "aug_prob" : 0.2 - } - } - }, - "training" : { - "//" : "out_filename: set to null for stdout. all path relative to save_base_dir", - "path" : "/path/to/stage3/train", - "ckpt_dir" : "ckpt", - "ckpt_reload" : "last.ckpt", - "model_onnx_filename" : "last.onnx", - "loggers" : { - "PrintLogger" : { "out_filename" : "logs/log.txt" }, - "ProgressLogger" : { "out_filename" : null }, - "TensorboardLogger" : { "log_dir" : "tensorboard" } - } - }, - "conversion" : { - "//" : "path to output the model. input model is taken in training/ckpt_dir", - "path" : "/path/to/stage3/train", - "model_filename" : "model_float.sadl" - }, - "conversion_int" : { - "//" : "path to output the model. input model is taken in training/ckpt_dir", - "path" : "/path/to/stage3/train", - "model_filename" : "model_int16.sadl" - } - } -} diff --git a/training/training_scripts/NN_Filtering_HOP/default_config.json b/training/training_scripts/NN_Filtering_HOP/default_config.json new file mode 100644 index 000000000..701ff57e0 --- /dev/null +++ b/training/training_scripts/NN_Filtering_HOP/default_config.json @@ -0,0 +1,670 @@ +{ + "verbose": 2, + "binaries": { + "ffmpeg": "ffmpeg", + "sadl_path": "/path/to/src/sadl" + }, + "dataset": { + "div2k_train": { + "//": "dataset of png to convert", + "start": 1, + "count": 800, + "path": "/path/to/DIV2K/DIV2K_train_HR" + }, + "div2k_valid": { + "start": 801, + "count": 100, + "path": "/path/to/DIV2K/DIV2K_valid_HR" + }, + "bvi": { + "//": "dataset of yuv", + "path": "/path/to/bviorg", + "dataset_file": "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/bvi.json" + }, + "bvi_valid": { + "//": "dataset of yuv", + "path": "/path/to/bviorg", + "dataset_file": "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/bvi_valid.json" + }, + "tvd": { + "//": "dataset of yuv", + "path": "/path/to/tvdorg", + "dataset_file": "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/tvd.json" + }, + "tvd_valid": { + "//": "dataset of yuv", + "path": "/path/to/tvdorg", + "dataset_file": "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/tvd_valid.json" + } + }, + + "model" : { + "class" : "Net", + "input_channels" : [ + [ + "rec_before_dbf_Y", + "rec_before_dbf_U", + "rec_before_dbf_V" + ], + [ + "pred_Y", + "pred_U", + "pred_V" + ], + [ + "bs_Y", + "bs_U", + "bs_V" + ], + [ "qp_base" ], + [ "qp_slice" ], + [ "ipb_Y" ] + ], + "input_kernels" : [ + 3, + 3, + 3, + 1, + 1, + 3 + ], + "D1" : 192, + "D2" : 32, + "D3" : 16, + "D4" : 16, + "D5" : 16, + "D6" : 48, + "N" : 24, + "C" : 64, + "C1" : 160, + "C21" : 32, + "C22" : 32, + "C31" : 64 + }, + + + "stage1": { + "yuv": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage1/yuv", + "dataset_file": "dataset.json" + }, + "yuv_valid": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage1/yuv", + "dataset_file": "dataset_valid.json" + }, + "encdec": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage1/encdec", + "dump_dir": "dump", + "enc_dir": "enc", + "dec_dir": "dec", + "cfg_dir": "cfg", + "qps": "19 22 24 27 29 32 34 37 39 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_intra_vtm.cfg", + "dataset_file": "dataset.json", + "encode_script": "encode_decode_dataset.sh" + }, + "encdec_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage1/encdec", + "dump_dir": "dump_valid", + "enc_dir": "enc_valid", + "dec_dir": "dec_valid", + "cfg_dir": "cfg_valid", + "qps": "19 22 24 27 29 32 34 37 39 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_intra_vtm.cfg", + "dataset_file": "dataset_valid.json", + "encode_script": "encode_decode_dataset_valid.sh" + }, + "dataset": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage1/dataset", + "dataset_file": "div2k.json" + }, + "dataset_valid": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage1/dataset", + "dataset_file": "div2k_valid.json" + }, + "training": { + "path": "/path/to/stage1/train", + "seed": 42, + "//": "additional loggers (e.g. for tensorboard) can be implemented in logger.py and added here", + "device": "cuda", + "multi_gpu": false, + "validation_interval": 1, + "mse_epoch": 37, + "max_epochs": 40, + "component_loss_weightings": [ + 6, + 2 + ], + "dataset_config": { + "class": "HOPDataset", + "quantizer": { + "org_Y": 1024, + "org_U": 1024, + "org_V": 1024, + "pred_Y": 1024, + "pred_U": 1024, + "pred_V": 1024, + "rec_before_dbf_Y": 1024, + "rec_before_dbf_U": 1024, + "rec_before_dbf_V": 1024, + "bs_Y": 1024, + "bs_U": 1024, + "bs_V": 1024, + "qp_base": 64, + "qp_slice": 64, + "ipb_Y": 2 + } + }, + "dataloader": { + "batch_size": 64, + "num_workers": 16, + "shuffle": true, + "pin_memory": true, + "drop_last": false + }, + "optimizer": { + "class": "Adam", + "lr": 0.0001, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-08, + "amsgrad": false + }, + "lr_scheduler": { + "class": "MultiStepLR", + "milestones": [ + 30, + 34 + ], + "gamma": 0.1, + "last_epoch": -1, + "verbose": false + }, + "model_ckpt": { + "//": "checkpoint training state regularly", + "save_last": true, + "export_last": true, + "every_n_epochs": 1 + }, + "loggers": { + "PrintLogger": { + "log_train_interval": 0, + "log_val_metrics": true + }, + "ProgressLogger": { + "out_filename": null, + "log_train_epochs": true, + "log_val_epochs": true, + "log_train_iterations": false, + "log_stage_ends": false + }, + "TensorboardLogger": { + "log_dir": "tensorboard", + "log_train_interval": 0, + "log_train_progress": true, + "log_val_metrics": true + } + } + }, + "conversion": { + "//": "path to output the model. input model is taken in training/ckpt_dir", + "path": "/path/to/stage1/train", + "model_filename": "model_float.sadl" + } + }, + "stage2": { + "yuv_tvd": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage2/yuv", + "dataset_file": "dataset_tvd.json" + }, + "yuv_tvd_valid": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage2/yuv", + "dataset_file": "dataset_tvd_valid.json" + }, + "yuv_bvi": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage2/yuv", + "dataset_file": "dataset_bvi.json" + }, + "yuv_bvi_valid": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage2/yuv", + "dataset_file": "dataset_bvi_valid.json" + }, + "encdec_bvi": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage2/encdec", + "dump_dir": "dump_bvi", + "enc_dir": "enc_bvi", + "dec_dir": "dec_bvi", + "cfg_dir": "cfg_bvi", + "qps": "22 27 32 37 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_option": "--NnlfHopDebugOption=1", + "vtm_model": "stage1/conversion", + "dataset_file": "dataset_bvi.json", + "encode_script": "encode_decode_dataset_bvi.sh" + }, + "encdec_bvi_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage2/encdec", + "dump_dir": "dump_bvi_valid", + "enc_dir": "enc_bvi_valid", + "dec_dir": "dec_bvi_valid", + "cfg_dir": "cfg_bvi_valid", + "qps": "22 27 32 37 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_option": "--NnlfHopDebugOption=1", + "vtm_model": "stage1/conversion", + "dataset_file": "dataset_bvi_valid.json", + "encode_script": "encode_decode_dataset_bvi_valid.sh" + }, + "encdec_tvd": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage2/encdec", + "dump_dir": "dump_tvd", + "enc_dir": "enc_tvd", + "dec_dir": "dec_tvd", + "cfg_dir": "cfg_tvd", + "qps": "22 27 32 37 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_option": "--NnlfHopDebugOption=1", + "vtm_model": "stage1/conversion", + "dataset_file": "dataset_tvd.json", + "encode_script": "encode_decode_dataset_tvd.sh" + }, + "encdec_tvd_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage2/encdec", + "dump_dir": "dump_tvd_valid", + "enc_dir": "enc_tvd_valid", + "dec_dir": "dec_tvd_valid", + "cfg_dir": "cfg_tvd_valid", + "qps": "22 27 32 37 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_option": "--NnlfHopDebugOption=1", + "vtm_model": "stage1/conversion", + "dataset_file": "dataset_tvd_valid.json", + "encode_script": "encode_decode_dataset_tvd_valid.sh" + }, + "dataset": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage2/dataset", + "dataset_files": { + "div2k": { + "dataset_file": "div2k.json", + "augs": [], + "aug_prob": 0.0 + }, + "bvi": { + "dataset_file": "bvi.json", + "augs": [ + "vflip", + "hflip", + "transpose" + ], + "aug_prob": 0.2 + }, + "tvd": { + "dataset_file": "tvd.json", + "augs": [ + "vflip", + "hflip", + "transpose" + ], + "aug_prob": 0.2 + } + } + }, + "dataset_valid": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage2/dataset", + "dataset_files": { + "div2k": { + "dataset_file": "div2k_valid.json", + "augs": [], + "aug_prob": 0.0 + }, + "bvi": { + "dataset_file": "bvi_valid.json", + "augs": [], + "aug_prob": 0.2 + }, + "tvd": { + "dataset_file": "tvd_valid.json", + "augs": [], + "aug_prob": 0.2 + } + } + }, + "training": { + "path": "/path/to/stage2/train", + "seed": 42, + "//": "additional loggers (e.g. for tensorboard) can be implemented in logger.py and added here", + "device": "cuda", + "multi_gpu": false, + "validation_interval": 1, + "mse_epoch": 18, + "max_epochs": 20, + "component_loss_weightings": [ + 6, + 2 + ], + "dataset_config": { + "class": "HOPDataset", + "quantizer": { + "org_Y": 1024, + "org_U": 1024, + "org_V": 1024, + "pred_Y": 1024, + "pred_U": 1024, + "pred_V": 1024, + "rec_before_dbf_Y": 1024, + "rec_before_dbf_U": 1024, + "rec_before_dbf_V": 1024, + "bs_Y": 1024, + "bs_U": 1024, + "bs_V": 1024, + "qp_base": 64, + "qp_slice": 64, + "ipb_Y": 2 + } + }, + "dataloader": { + "batch_size": 64, + "num_workers": 16, + "shuffle": true, + "pin_memory": true, + "drop_last": false + }, + "optimizer": { + "class": "Adam", + "lr": 0.0001, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-08, + "amsgrad": false + }, + "lr_scheduler": { + "class": "MultiStepLR", + "milestones": [ + 15, + 17 + ], + "gamma": 0.1, + "last_epoch": -1, + "verbose": false + }, + "model_ckpt": { + "//": "checkpoint training state regularly", + "save_last": true, + "export_last": true, + "every_n_epochs": 1 + }, + "loggers": { + "PrintLogger": { + "log_train_interval": 0, + "log_val_metrics": true + }, + "ProgressLogger": { + "out_filename": null, + "log_train_epochs": true, + "log_val_epochs": true, + "log_train_iterations": false, + "log_stage_ends": false + }, + "TensorboardLogger": { + "log_dir": "tensorboard", + "log_train_interval": 0, + "log_train_progress": true, + "log_val_metrics": true + } + } + }, + "conversion": { + "//": "path to output the model. input model is taken in training/ckpt_dir", + "path": "/path/to/stage2/train", + "model_filename": "model_float.sadl" + } + }, + "stage3": { + "encdec_bvi": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage3/encdec", + "dump_dir": "dump_bvi", + "enc_dir": "enc_bvi", + "dec_dir": "dec_bvi", + "cfg_dir": "cfg_bvi", + "qps": "22 27 32 37 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_option": "", + "vtm_model": "stage2/conversion", + "dataset_file": "dataset_bvi.json", + "encode_script": "encode_decode_dataset_bvi.sh" + }, + "encdec_bvi_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage3/encdec", + "dump_dir": "dump_bvi_valid", + "enc_dir": "enc_bvi_valid", + "dec_dir": "dec_bvi_valid", + "cfg_dir": "cfg_bvi_valid", + "qps": "22 27 32 37 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_option": "", + "vtm_model": "stage2/conversion", + "dataset_file": "dataset_bvi_valid.json", + "encode_script": "encode_decode_dataset_bvi_valid.sh" + }, + "encdec_tvd": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage3/encdec", + "dump_dir": "dump_tvd", + "enc_dir": "enc_tvd", + "dec_dir": "dec_tvd", + "cfg_dir": "cfg_tvd", + "qps": "22 27 32 37 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_option": "", + "vtm_model": "stage2/conversion", + "dataset_file": "dataset_tvd.json", + "encode_script": "encode_decode_dataset_tvd.sh" + }, + "encdec_tvd_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage3/encdec", + "dump_dir": "dump_tvd_valid", + "enc_dir": "enc_tvd_valid", + "dec_dir": "dec_tvd_valid", + "cfg_dir": "cfg_tvd_valid", + "qps": "22 27 32 37 42", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_option": "", + "vtm_model": "stage2/conversion", + "dataset_file": "dataset_tvd_valid.json", + "encode_script": "encode_decode_dataset_tvd_valid.sh" + }, + "dataset": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage3/dataset", + "dataset_files": { + "div2k": { + "dataset_file": "div2k.json", + "augs": [], + "aug_prob": 0.0 + }, + "bvi": { + "dataset_file": "bvi.json", + "augs": [ + "vflip", + "hflip", + "transpose" + ], + "aug_prob": 0.2 + }, + "tvd": { + "dataset_file": "tvd.json", + "augs": [ + "vflip", + "hflip", + "transpose" + ], + "aug_prob": 0.2 + } + } + }, + "dataset_valid": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage3/dataset", + "dataset_files": { + "div2k": { + "dataset_file": "div2k_valid.json", + "augs": [], + "aug_prob": 0.0 + }, + "bvi": { + "dataset_file": "bvi_valid.json", + "augs": [], + "aug_prob": 0.2 + }, + "tvd": { + "dataset_file": "tvd_valid.json", + "augs": [], + "aug_prob": 0.2 + } + } + }, + "training": { + "path": "/path/to/stage3/train", + "seed": 42, + "//": "additional loggers (e.g. for tensorboard) can be implemented in logger.py and added here", + "device": "cuda", + "multi_gpu": false, + "validation_interval": 1, + "mse_epoch": 18, + "max_epochs": 20, + "component_loss_weightings": [ + 6, + 2 + ], + "dataset_config": { + "class": "HOPDataset", + "quantizer": { + "org_Y": 1024, + "org_U": 1024, + "org_V": 1024, + "pred_Y": 1024, + "pred_U": 1024, + "pred_V": 1024, + "rec_before_dbf_Y": 1024, + "rec_before_dbf_U": 1024, + "rec_before_dbf_V": 1024, + "bs_Y": 1024, + "bs_U": 1024, + "bs_V": 1024, + "qp_base": 64, + "qp_slice": 64, + "ipb_Y": 2 + } + }, + "dataloader": { + "batch_size": 64, + "num_workers": 16, + "shuffle": true, + "pin_memory": true, + "drop_last": false + }, + "optimizer": { + "class": "Adam", + "lr": 0.0001, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-08, + "amsgrad": false + }, + "lr_scheduler": { + "class": "MultiStepLR", + "milestones": [ + 15, + 17, + 18 + ], + "gamma": 0.1, + "last_epoch": -1, + "verbose": false + }, + "model_ckpt": { + "//": "checkpoint training state regularly", + "save_last": true, + "export_last": true, + "every_n_epochs": 1 + }, + "loggers": { + "PrintLogger": { + "log_train_interval": 0, + "log_val_metrics": true + }, + "ProgressLogger": { + "out_filename": null, + "log_train_epochs": true, + "log_val_epochs": true, + "log_train_iterations": false, + "log_stage_ends": false + }, + "TensorboardLogger": { + "log_dir": "tensorboard", + "log_train_interval": 0, + "log_train_progress": true, + "log_val_metrics": true + } + } + }, + "conversion": { + "//": "path to output the model. input model is taken in training/ckpt_dir", + "path": "/path/to/stage3/train", + "model_filename": "model_float.sadl" + }, + "conversion_int": { + "//": "path to output the model. input model is taken in training/ckpt_dir", + "path": "/path/to/stage3/train", + "model_filename": "model_int16.sadl" + } + } +} diff --git a/training/training_scripts/NN_Filtering_HOP/paths.json b/training/training_scripts/NN_Filtering_HOP/paths.json new file mode 100644 index 000000000..7dc8990bb --- /dev/null +++ b/training/training_scripts/NN_Filtering_HOP/paths.json @@ -0,0 +1,192 @@ +{ + "binaries": { + "sadl_path": "/path/to/src/sadl" + }, + + "dataset": { + "div2k_train": { + "//": "dataset of png to convert", + "path": "/path/to/DIV2K/DIV2K_train_HR" + }, + "div2k_valid": { + "path": "/path/to/DIV2K/DIV2K_valid_HR" + }, + "bvi": { + "//": "dataset of yuv", + "path": "/path/to/bviorg", + "dataset_file": "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/bvi.json" + }, + "bvi_valid": { + "//": "dataset of yuv", + "path": "/path/to/bviorg", + "dataset_file": "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/bvi_valid.json" + }, + "tvd": { + "//": "dataset of yuv", + "path": "/path/to/tvdorg", + "dataset_file": "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/tvd.json" + }, + "tvd_valid": { + "//": "dataset of yuv", + "path": "/path/to/tvdorg", + "dataset_file": "/path/to/src/training/training_scripts/NN_Filtering_HOP/datasets/tvd_valid.json" + } + }, + + + "stage1": { + "yuv": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage1/yuv" + }, + "yuv_valid": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage1/yuv" + }, + "encdec": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage1/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_intra_vtm.cfg" + }, + "encdec_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage1/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_intra_vtm.cfg" + }, + "dataset": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage1/dataset" + }, + "dataset_valid": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage1/dataset" + }, + "training": { + "path": "/path/to/stage1/train" + }, + "conversion": { + "//": "path to output the model. input model is taken in training/ckpt_dir", + "path": "/path/to/stage1/train" + } + }, + + + "stage2": { + "yuv_tvd": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage2/yuv" + }, + "yuv_tvd_valid": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage2/yuv" + }, + "yuv_bvi": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage2/yuv" + }, + "yuv_bvi_valid": { + "//": "path to store yuv files dataset", + "path": "/path/to/stage2/yuv" + }, + "encdec_bvi": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage2/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg" + }, + "encdec_bvi_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage2/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg" + }, + "encdec_tvd": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage2/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg" + }, + "encdec_tvd_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage2/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg" + }, + "dataset": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage2/dataset", + }, + "dataset_valid": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage2/dataset" + }, + "training": { + "path": "/path/to/stage2/train" + }, + "conversion": { + "//": "path to output the model. input model is taken in training/ckpt_dir", + "path": "/path/to/stage2/train" + } + }, + + + "stage3": { + "encdec_bvi": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage3/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + }, + "encdec_bvi_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage3/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + }, + "encdec_tvd": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage3/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + }, + "encdec_tvd_valid": { + "//": "path to store the shell script and all the generated files by the encoder/decoder", + "path": "/path/to/stage3/encdec", + "vtm_enc": "/path/to/src/bin/EncoderAppStatic", + "vtm_dec": "/path/to/src/bin/DecoderAppStatic", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + }, + "dataset": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage3/dataset" + }, + "dataset_valid": { + "//": "path to store the full dataset which will be used by the training", + "path": "/path/to/stage3/dataset" + }, + "training": { + "path": "/path/to/stage3/train" + }, + "conversion": { + "//": "path to output the model. input model is taken in training/ckpt_dir", + "path": "/path/to/stage3/train", + }, + "conversion_int": { + "//": "path to output the model. input model is taken in training/ckpt_dir", + "path": "/path/to/stage3/train", + } + } + +} + diff --git a/training/training_scripts/NN_Filtering_HOP/training/cfg/debug.json b/training/training_scripts/NN_Filtering_HOP/training/cfg/debug.json deleted file mode 100644 index 9ec4558ff..000000000 --- a/training/training_scripts/NN_Filtering_HOP/training/cfg/debug.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "tag": "debug", - "mse_epoch": 8, - "max_epochs": 10, - "device": "cpu", - "dataset": { - "class": "MockDataset", - "length": 32 - }, - "dataloader": { - "batch_size": 8, - "num_workers": 8 - }, - "lr_scheduler": { - "milestones": [4, 6, 8] - }, - "loggers": { - "PrintLogger": { - "out_filename": "./logs/debug/log.txt" - } - } -} \ No newline at end of file diff --git a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage1.json b/training/training_scripts/NN_Filtering_HOP/training/cfg/stage1.json deleted file mode 100644 index bff5c2ce3..000000000 --- a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage1.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "current_stage" : "stage1", - "verbose" : 2, - "stage1" : { - "training" : { - "//" : "the epoch to switch from L1 to L2 loss, default: 80, max_epochs: 100", - "mse_epoch" : 37, - "max_epochs" : 40, - "lr_scheduler" : { - "milestones" : [ - 30, - 34 - ] - } - } - } -} \ No newline at end of file diff --git a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage1_fast.json b/training/training_scripts/NN_Filtering_HOP/training/cfg/stage1_fast.json deleted file mode 100644 index 7945dd185..000000000 --- a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage1_fast.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "current_stage" : "stage1", - "verbose" : 2, - "stage1" : { - "training" : { - "//" : "the epoch to switch from L1 to L2 loss, default: 80, max_epochs: 100", - "mse_epoch" : 11, - "max_epochs" : 12, - "lr_scheduler" : { - "milestones" : [ - 9, - 10 - ] - } - } - } -} diff --git a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage2.json b/training/training_scripts/NN_Filtering_HOP/training/cfg/stage2.json deleted file mode 100644 index 2a8516beb..000000000 --- a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage2.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "current_stage" : "stage2", - "verbose" : 2, - "stage2" : { - "training" : { - "//" : "the epoch to switch from L1 to L2 loss, default: 80, max_epochs: 100", - "mse_epoch" : 18, - "max_epochs" : 20, - "lr_scheduler" : { - "milestones" : [ - 15, - 17 - ] - } - } - } -} - diff --git a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage2_fast.json b/training/training_scripts/NN_Filtering_HOP/training/cfg/stage2_fast.json deleted file mode 100644 index b846b629e..000000000 --- a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage2_fast.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "current_stage" : "stage2", - "verbose" : 2, - "stage2" : { - "training" : { - "//" : "the epoch to switch from L1 to L2 loss, default: 80, max_epochs: 100", - "mse_epoch" : 8, - "max_epochs" : 10, - "lr_scheduler" : { - "milestones" : [ - 6, - 7 - ] - } - } - } -} diff --git a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage3.json b/training/training_scripts/NN_Filtering_HOP/training/cfg/stage3.json deleted file mode 100644 index 49de105fb..000000000 --- a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage3.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "current_stage" : "stage3", - "verbose" : 2, - "stage3" : { - "training" : { - "//" : "the epoch to switch from L1 to L2 loss, default: 80, max_epochs: 100", - "mse_epoch" : 18, - "max_epochs" : 20, - "override_scheduler" : false, - "lr_scheduler" : { - "milestones" : [ - 15, - 17, - 18 - ] - } - } - } -} - diff --git a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage3_fast.json b/training/training_scripts/NN_Filtering_HOP/training/cfg/stage3_fast.json deleted file mode 100644 index 2c75d842a..000000000 --- a/training/training_scripts/NN_Filtering_HOP/training/cfg/stage3_fast.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "current_stage" : "stage3", - "verbose" : 2, - "stage3" : { - "training" : { - "//" : "the epoch to switch from L1 to L2 loss, default: 80, max_epochs: 100", - "mse_epoch" : 8, - "max_epochs" : 10, - "lr_scheduler" : { - "milestones" : [ - 6, - 7 - ] - } - } - } -} diff --git a/training/training_scripts/NN_Filtering_HOP/training/cfg/training_default.json b/training/training_scripts/NN_Filtering_HOP/training/cfg/training_default.json deleted file mode 100644 index fcbec2518..000000000 --- a/training/training_scripts/NN_Filtering_HOP/training/cfg/training_default.json +++ /dev/null @@ -1,85 +0,0 @@ -{ "training" : { - "seed" : 42, - "//" : "additional loggers (e.g. for tensorboard) can be implemented in logger.py and added here", - "device" : "cuda", - "multi_gpu" : false, - "validation_interval" : 1, - "max_epochs" : 100, - "mse_epoch" : 90, - "component_loss_weightings" : [ - 6, - 2 - ], - "dataset_config" : { - "class" : "HOPDataset", - "quantizer" : { - "org_Y" : 1024, - "org_U" : 1024, - "org_V" : 1024, - "pred_Y" : 1024, - "pred_U" : 1024, - "pred_V" : 1024, - "rec_before_dbf_Y" : 1024, - "rec_before_dbf_U" : 1024, - "rec_before_dbf_V" : 1024, - "bs_Y" : 1024, - "bs_U" : 1024, - "bs_V" : 1024, - "qp_base" : 64, - "qp_slice" : 64, - "ipb_Y" : 2 - } - }, - "dataloader" : { - "batch_size" : 64, - "num_workers" : 16, - "shuffle" : true, - "pin_memory" : true, - "drop_last" : false - }, - "optimizer" : { - "class" : "Adam", - "lr" : 0.0001, - "betas" : [ - 0.9, - 0.999 - ], - "eps" : 1e-08, - "weight_decay" : 1e-08, - "amsgrad" : false - }, - "lr_scheduler" : { - "class" : "MultiStepLR", - "milestones" : [ - 40, - 60, - 80 - ], - "gamma" : 0.1, - "last_epoch" : -1, - "verbose" : false - }, - "model_ckpt" : { - "//" : "checkpoint training state regularly", - "save_last" : true, - "export_last" : true, - "every_n_epochs" : 1 - }, - "loggers" : { - "PrintLogger" : { - "log_train_interval" : 0, - "log_val_metrics" : true - }, - "ProgressLogger" : { - "log_train_epochs" : true, - "log_val_epochs" : true, - "log_train_iterations" : false, - "log_stage_ends" : false - }, - "TensorboardLogger" : { - "log_train_progress" : true, - "log_train_interval" : 0, - "log_val_metrics" : true - } - } - } } diff --git a/training/training_scripts/NN_Filtering_HOP/training/main.py b/training/training_scripts/NN_Filtering_HOP/training/main.py index 4847a9194..0b05f67c1 100644 --- a/training/training_scripts/NN_Filtering_HOP/training/main.py +++ b/training/training_scripts/NN_Filtering_HOP/training/main.py @@ -32,34 +32,38 @@ * THE POSSIBILITY OF SUCH DAMAGE. """ -import sys +import argparse import json -import jsonmerge from trainer import Trainer +parser = argparse.ArgumentParser( + prog="train HOP model", + usage="main.py --json_config my_config.json --stage N\nN: stage number in [1-3]", + formatter_class=argparse.RawDescriptionHelpFormatter, +) -# CLI: python main.py [CONFIG]... -config = {"verbose": 0} -default_training = {} -for config_filename in sys.argv[1:]: - with open(config_filename, "r") as file: - j = json.load(file) - if "training" in j: - default_training = j - else: - config = jsonmerge.merge(config, j) +parser.add_argument( + "--json_config", + action="store", + type=str, + help="global configuration file", + required=True, +) + +parser.add_argument( + "--stage", action="store", type=int, help="stage number in [1-3]", required=True +) + +args = parser.parse_args() +try: + with open(args.json_config) as file: + config = json.load(file) +except Exception: + quit("[ERROR] unable to open json config") -if "current_stage" in config: - print(f"[INFO] training for {config['current_stage']}") - if "training" in default_training: - config[config["current_stage"]] = jsonmerge.merge( - default_training, config[config["current_stage"]] - ) -else: - quit("[ERROR] no current stage found in config files") if config["verbose"] > 1: print(json.dumps(config, indent=1)) -trainer = Trainer(config) +trainer = Trainer(config, args.stage) trainer.train() diff --git a/training/training_scripts/NN_Filtering_HOP/training/trainer.py b/training/training_scripts/NN_Filtering_HOP/training/trainer.py index 022093314..070b471f0 100644 --- a/training/training_scripts/NN_Filtering_HOP/training/trainer.py +++ b/training/training_scripts/NN_Filtering_HOP/training/trainer.py @@ -52,13 +52,13 @@ import model # noqa: E402 class Trainer: """Training procedure.""" - def __init__(self, config: Dict[str, Any]): + def __init__(self, config: Dict[str, Any], stage: int): """ Args: config: config hyperparameters stored in dictionary. See cfg/base_desc.txt for more details. """ self.config = config - self.config_stage = config[config["current_stage"]] + self.config_stage = f"stage{stage}" self.config_training = self.config_stage["training"] self.current_epoch = 0 self.current_iteration = 0 -- GitLab From 511d5f34c5269ab5ed1fb5e86a860948731a52f8 Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Thu, 3 Aug 2023 12:12:17 +0200 Subject: [PATCH 02/12] unified scripts API --- .../NN_Filtering_HOP/default_config.json | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/training/training_scripts/NN_Filtering_HOP/default_config.json b/training/training_scripts/NN_Filtering_HOP/default_config.json index 701ff57e0..b6c8c518a 100644 --- a/training/training_scripts/NN_Filtering_HOP/default_config.json +++ b/training/training_scripts/NN_Filtering_HOP/default_config.json @@ -38,49 +38,6 @@ } }, - "model" : { - "class" : "Net", - "input_channels" : [ - [ - "rec_before_dbf_Y", - "rec_before_dbf_U", - "rec_before_dbf_V" - ], - [ - "pred_Y", - "pred_U", - "pred_V" - ], - [ - "bs_Y", - "bs_U", - "bs_V" - ], - [ "qp_base" ], - [ "qp_slice" ], - [ "ipb_Y" ] - ], - "input_kernels" : [ - 3, - 3, - 3, - 1, - 1, - 3 - ], - "D1" : 192, - "D2" : 32, - "D3" : 16, - "D4" : 16, - "D5" : 16, - "D6" : 48, - "N" : 24, - "C" : 64, - "C1" : 160, - "C21" : 32, - "C22" : 32, - "C31" : 64 - }, "stage1": { -- GitLab From 2050f20242f6b3c950103c77edc375923d25fbbc Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Thu, 10 Aug 2023 10:41:09 +0200 Subject: [PATCH 03/12] clarify readme for partial retraining and NNVC version --- .../NN_Filtering_HOP/readme.md | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/training/training_scripts/NN_Filtering_HOP/readme.md b/training/training_scripts/NN_Filtering_HOP/readme.md index 61a2f601c..a38102934 100644 --- a/training/training_scripts/NN_Filtering_HOP/readme.md +++ b/training/training_scripts/NN_Filtering_HOP/readme.md @@ -26,7 +26,7 @@ dataset files are placed in the target directory (as set in the config.json ["st ** Note ** After conversion, the dataset size is about 21GB. #### 2. Prepare scripts for encoding/decoding of the dataset -Please note that a VTM without NN tools is used. NNVC-5.0 or NNVC-4.0 tags can be used to generate the binaries and cfg file. The configuration file is the vanilla VTM one (see config.json). +Please note that a VTM without NN tools is used. NNVC-4.0 to NNVC-6.0 tags can be used to generate the binaries and cfg file. The configuration file is the vanilla VTM one (see config.json). The macro for data dump should be: ``` // which data are used for inference/dump @@ -177,7 +177,14 @@ The flag ``NnlfHopDebugOption`` is also needed at decoder since it forces the us ## II- Model Stage 2 Total size required for stage2 (without deleting intermediate data) is about 5TB. + + ### A- Data extraction + +##### 0. Training from stage II +An official model at the end of stage I is available in the directory ``models`` in order to train directly from stage II (in order to skip stage I). +To do so, the model ``HOP_float.I.sadl`` is used to generate the dataset for stage II. Please replace the model path by the one of ``HOP_float.I.sadl`` in your ``config.json`` in order to use the official model. + #### 1. Dataset preparation - bvi/tvd conversion Convert BVI-DVC 64 frames sequence into 65 frames sequence, and create the TVD dataset file (YUV are still taken from source). @@ -296,11 +303,19 @@ python3 training_scripts/NN_Filtering_HOP/convert/to_sadl.py --json_config tra **Note:** the directory in dataset can now be deleted if there is no need to retrain. -#### 3. Test model -To test the float model, the type should be set as float in ``TypeDef.h``: +#### 3. Integerize model +The float model is integerized into int16 SADL format using a naive quantization (all quantizers to 2^11). +First the ``naive_quantization`` software should be build. Please refer to SADL documentation to build the software. +```sh +python3 training_scripts/NN_Filtering_HOP/quantize/quantize.py | naive_quantization model_float.sadl model_int16.sadl +``` + + +#### 4. Test model +To test the int16 model, the type should be set as int16 in ``TypeDef.h``: ``` #if NN_HOP_UNIFIED -using TypeSadlHOP=float; +using TypeSadlHOP=int16_t; #endif ``` and the correct data macros should be set: @@ -315,9 +330,9 @@ and the correct data macros should be set: The model is tested with NNVC-5.1 using the following parameters: ``` ---NnlfOption=4 --NnlfHopModelName=stage2/train/model_float.sadl +--NnlfOption=4 --NnlfHopModelName=stage2/train/model_int16.sadl ``` -The configuration file is ``encoder_xxx_vtm.cfg`` and the anchor VTM-11.0\_NNVC. +The configuration file is ``encoder_xxx_nnvc.cfg`` and the anchor VTM-11.0\_NNVC v5.1. @@ -329,11 +344,16 @@ The configuration file is ``encoder_xxx_vtm.cfg`` and the anchor VTM-11.0\_NNVC. ## III- Model Stage 3 ### A- Data extraction #### 1. Dataset preparation +##### 1.1 All previously prepared datasets will be reused for stage 3. +##### 1.2 Training from stage III +An official model at the end of stage II is available in the directory ``models`` in order to train directly from stage III. +To do so, the model ``HOP_int16.II.sadl`` is used to generate the dataset for stage III. Please replace the model path by the one of ``HOP_int16.II.sadl`` in your ``config.json`` in order to use the official model. Checksums for the resulting dataset are also available in the nnvc-ctc repository. + #### 2. Prepare script for encoding/decoding of the dataset -This step is similar to II-A-2, only the used model is different. -Please note that a VTM with the model of stqge 1 is used. NNVC-5.1 can be used to generate the binaries and cfg file. The configuration file is the vanilla VTM one (see config.json), the input model is added automatically. +This step is similar to II-A-2, only the used model is different. Please note that the int16 model is used to generate the dataset. +Please note that a VTM with the model of stage 1 is used. NNVC-5.1 can be used to generate the binaries and cfg file. The configuration file is the vanilla VTM one (see config.json), the input model is added automatically. The macro for data dump should be: ``` // which data are used for inference/dump @@ -425,18 +445,18 @@ python3 training_scripts/NN_Filtering_HOP/convert/to_sadl.py --json_config trai --input_model stage3/training --output_model stage3/conversion ``` -#### 3. Integerize model -The float model is integerized into int16 SADL format. +#### 3. Integerized model +The float model is integerized into int16 SADL format using a naive quantization (all quantizers to 2^11). +First the ``naive_quantization`` software should be build. Please refer to SADL documentation to build the software. ```sh -python3 training_scripts/NN_Filtering_HOP/convert/quantize.py --json_config training_scripts/NN_Filtering_HOP/config.json \ - --input_model stage3/conversion --output_model stage3/conversion_int --quantize training_scripts/NN_Filtering_HOP/convert/quantize.json +python3 training_scripts/NN_Filtering_HOP/quantize/quantize.py | naive_quantization model_float.sadl model_int16.sadl ``` **Note:** the directory in dataset can now be deleted if there is no need to retrain. -#### 3. Test model -To test the int16 model, the type should be set as in16 in ``TypeDef.h``: +#### 4. Test model +To test the int16 model, the type should be set as int16 in ``TypeDef.h``: ``` #if NN_HOP_UNIFIED using TypeSadlHOP=int16_t; @@ -452,11 +472,11 @@ and the correct data macros should be set: #define JVET_AC0089_NNVC_USE_BPM_INFO 1 // JVET-AC0089: dump Block Prediction Mode ``` -The model is tested with NNVC-5.1 using the following parameters: +The model is tested with the last NNVC version using the following parameters: ``` --NnlfOption=4 --NnlfHopModelName=stage2/train/model_int16.sadl ``` -The configuration file is ``encoder_xxx_vtm.cfg`` and the anchor VTM-11.0\_NNVC. +The configuration file is ``encoder_xxx_nnvc.cfg`` and the anchor the last NNVC version. ## IV-Advanced -- GitLab From f94ae713b69448a61cfcf84a28724d22b20b0b97 Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Thu, 10 Aug 2023 09:06:52 +0000 Subject: [PATCH 04/12] Dev v6 --- source/Lib/EncoderLib/EncNNFilterHOP.cpp | 11 ++++++----- source/Lib/EncoderLib/EncNNFilterHOP.h | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/Lib/EncoderLib/EncNNFilterHOP.cpp b/source/Lib/EncoderLib/EncNNFilterHOP.cpp index daed59680..f07cba8a5 100644 --- a/source/Lib/EncoderLib/EncNNFilterHOP.cpp +++ b/source/Lib/EncoderLib/EncNNFilterHOP.cpp @@ -135,26 +135,26 @@ void EncNNFilterHOP::chooseParameters(Picture &pic) } double minCost = MAX_DOUBLE, minScaledCost = MAX_DOUBLE; + std::vector<int> bestPrmId(picprms.prmId); // find best parameters without scaling and store them - parameterSearch(pic, minCost, -1); + parameterSearch(pic, minCost, bestPrmId, -1); int mode = picprms.sprm.mode; - std::vector<int> prmId (picprms.prmId); // find best parameters with scaling for (int scaleId = 0; scaleId < 4; scaleId++) { scalePicture(pic, scaleId); - parameterSearch(pic, minScaledCost, scaleId); + parameterSearch(pic, minScaledCost, bestPrmId, scaleId); } // compare w/ scaling and w/o scaling if (minCost <= minScaledCost) { - picprms.prmId = prmId; picprms.sprm.mode = mode; picprms.sprm.scaleFlag = -1; } + picprms.prmId = bestPrmId; if (picprms.sprm.mode < picprms.prmNum) { @@ -190,7 +190,7 @@ void EncNNFilterHOP::chooseParameters(Picture &pic) } } -void EncNNFilterHOP::parameterSearch(Picture& pic, double& minCost, int scaleId) +void EncNNFilterHOP::parameterSearch(Picture &pic, double &minCost, std::vector<int> &bestPrmId, int scaleId) { CodingStructure& cs = *pic.cs; const PreCalcValues& pcv = *cs.pcv; @@ -263,6 +263,7 @@ void EncNNFilterHOP::parameterSearch(Picture& pic, double& minCost, int scaleId) { picprms.sprm.scaleFlag = scaleId + 4; } + bestPrmId = picprms.prmId; } } } diff --git a/source/Lib/EncoderLib/EncNNFilterHOP.h b/source/Lib/EncoderLib/EncNNFilterHOP.h index 64bd24f60..a7aa16b0f 100644 --- a/source/Lib/EncoderLib/EncNNFilterHOP.h +++ b/source/Lib/EncoderLib/EncNNFilterHOP.h @@ -56,7 +56,7 @@ public: private: void scaleFactorDerivation(Picture &pic, FilterParameters &prms, int prmId); void scalePicture(Picture &pic, int scaleId); - void parameterSearch(Picture &pic, double& minCost, int scaleId); + void parameterSearch(Picture &pic, double& minCost, std::vector<int> &bestPrmId, int scaleId); double getSignalingCost(Picture &pic); CABACWriter *m_CABACEstimator = nullptr; -- GitLab From ca925b7638f63218d94e39a499569a16afaa340a Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Fri, 11 Aug 2023 10:00:59 +0000 Subject: [PATCH 05/12] quantizer scripts and doc --- .../NN_Filtering_HOP/quantize/quantize.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 training/training_scripts/NN_Filtering_HOP/quantize/quantize.py diff --git a/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py b/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py new file mode 100644 index 000000000..42df4440a --- /dev/null +++ b/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py @@ -0,0 +1,109 @@ +""" +/* The copyright in this software is being made available under the BSD +* License, included below. This software may be subject to other third party +* and contributor rights, including patent rights, and no such rights are +* granted under this license. +* +* Copyright (c) 2010-2023, ITU/ISO/IEC +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are met: +* +* * Redistributions of source code must retain the above copyright notice, +* this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright notice, +* this list of conditions and the following disclaimer in the documentation +* and/or other materials provided with the distribution. +* * Neither the name of the ITU/ISO/IEC nor the names of its contributors may +* be used to endorse or promote products derived from this software without +* specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS +* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +* THE POSSIBILITY OF SUCH DAMAGE. +""" + +# inputs: REC, PRED, BS, QPb, QPs, IBP +Q = 11 +print(f"0 {Q} 1 {Q} 2 {Q} 3 {Q} 4 {Q} 5 {Q} ", end="") + +# conv2D + biais on inputs +Q = 11 +print( + f"6 {Q} 8 {Q} 12 {Q} 14 {Q} 18 {Q} 20 {Q} 24 {Q} 26 {Q} 30 {Q} 32 {Q} 36 {Q} 38 {Q} ", + end="", +) + +# conv2D internal quantization offset +L = [7, 13, 19, 25, 31, 37, 45, 51] +for i in L: + print(f" {i} 0 ", end="") + +# leaky relu +Q = 11 +L = [10, 16, 22, 28, 34, 40, 48, 54] +for i in L: + print(f" {i} {Q} ", end="") + + +# concatenate result NO QUANTIZATION on attribute +print(" 42 0 ", end="") + +# conv2D+bias channel reduction +Q = 11 +print(f" 44 {Q} 46 {Q} ", end="") + +# conv2d+bias +Q = 11 +print(f" 50 {Q} 52 {Q} ", end="") + + +# building blocks +Q = 11 +for i in range(24): + # conv internal quantization offset + print(f"{57+i*31} 0 ", end="") + print(f"{63+i*31} 0 ", end="") + print(f"{67+i*31} 0 ", end="") + print(f"{75+i*31} 0 ", end="") + print(f"{79+i*31} 0 ", end="") + print(f"{83+i*31} 0 ", end="") + + print(f"{72+i*31} 0 ", end="") + + # conv+bias + print(f"{56+i*31} {Q} ", end="") + print(f"{58+i*31} {Q} ", end="") + print(f"{62+i*31} {Q} ", end="") + print(f"{64+i*31} {Q} ", end="") + print(f"{66+i*31} {Q} ", end="") + print(f"{68+i*31} {Q} ", end="") + print(f"{74+i*31} {Q} ", end="") + print(f"{76+i*31} {Q} ", end="") + print(f"{78+i*31} {Q} ", end="") + print(f"{80+i*31} {Q} ", end="") + print(f"{82+i*31} {Q} ", end="") + print(f"{84+i*31} {Q} ", end="") + + # leaky relu + print(f"{70+i*31} {Q} ", end="") + print(f"{60+i*31} {Q} ", end="") + +# final conv+bias +Q = 11 +print(f" 800 {Q} 802 {Q} 806 {Q} 808 {Q}", end="") + +# leaky relu +print(f" 804 {Q} ", end="") + +# conv internal quantization offset +print(" 801 0 807 0 ") -- GitLab From 7a99fe1cd5ae1c04c0e36aa380b22e78ca459cc8 Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Thu, 10 Aug 2023 10:59:37 +0200 Subject: [PATCH 06/12] missing license header --- training/training_scripts/NN_Filtering_HOP/quantize/quantize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py b/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py index 42df4440a..7f0edea1a 100644 --- a/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py +++ b/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py @@ -31,7 +31,6 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. """ - # inputs: REC, PRED, BS, QPb, QPs, IBP Q = 11 print(f"0 {Q} 1 {Q} 2 {Q} 3 {Q} 4 {Q} 5 {Q} ", end="") -- GitLab From 435b4ee957cde2432a674160310b16a841ca8a73 Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Thu, 10 Aug 2023 10:15:18 +0200 Subject: [PATCH 07/12] quantizer scripts and doc --- .../training_scripts/NN_Filtering_HOP/quantize/quantize.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py b/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py index 7f0edea1a..a26e0da81 100644 --- a/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py +++ b/training/training_scripts/NN_Filtering_HOP/quantize/quantize.py @@ -31,6 +31,10 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. """ +<<<<<<< HEAD +======= + +>>>>>>> 7ef3a6a8 (quantizer scripts and doc) # inputs: REC, PRED, BS, QPb, QPs, IBP Q = 11 print(f"0 {Q} 1 {Q} 2 {Q} 3 {Q} 4 {Q} 5 {Q} ", end="") -- GitLab From 9c92879df97fa1b9074623c1a5248413f2f089ed Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Fri, 11 Aug 2023 11:57:11 +0200 Subject: [PATCH 08/12] update readme with new api + more info --- .../NN_Filtering_HOP/readme.md | 153 ++++++++++++------ 1 file changed, 107 insertions(+), 46 deletions(-) diff --git a/training/training_scripts/NN_Filtering_HOP/readme.md b/training/training_scripts/NN_Filtering_HOP/readme.md index a38102934..de9d12918 100644 --- a/training/training_scripts/NN_Filtering_HOP/readme.md +++ b/training/training_scripts/NN_Filtering_HOP/readme.md @@ -1,13 +1,84 @@ # High Operating Point model training ## Overview -First edit the file ``training_scripts/NN_Filtering_HOP/config.json`` to adapt all the paths. -All key with the name ``path`` should be edited to fit your particular environement. + +### Preparation of the directory +At minima, paths should first be set. The file ``training/training_scripts/NN_Filtering_HOP/paths.json`` should be edited to match your environment. +All keys with the name ``path`` should be edited to fit your particular environement. Additionally, you should also edit the variable ``vtm_xx`` to point to the VTM binaries and configuration files, the ``sadl_path`` to point to the sadl repository. -Other keys like filenames can be let as is, except for debugging purpose. - Once the paths are setup, you should be able to run the process just by copy/pasting all lines of shell below. +Assuming that all data are on the same storage, then the following directory structure can be used: +``` + - src [ contains the NNVC repository] + - DIV2K [ contains the original div2k dataset] + - bviorg [ contains the original BVI dataset ] + - tvdorg [ contains the original TVD dataset ] + + - stage1 [ will be created by the scripts] + - stage2 [ will be created by the scripts] + - stage3 [ will be created by the scripts] +``` + +To create ``src`` the following commands can be used: +```sh +git clone https://vcgit.hhi.fraunhofer.de/jvet-ahg-nnvc/VVCSoftware_VTM.git src; +cd src; +git checkout VTM-11.0_nnvc +git submodule init +git submodule update +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=Release .. +make -j +cd ../.. +``` + +To create the DIV2K directory: +```sh +mkdir DIV2K; +cd DIV2K; +wget http://data.vision.ee.ethz.ch/cvl/DIV2K/DIV2K_train_HR.zip +unzip DIV2K_train_HR.zip +wget http://data.vision.ee.ethz.ch/cvl/DIV2K/DIV2K_valid_HR.zip +unzip DIV2K_valid_HR.zip +cd .. +``` + +To create the bviorg directory: +```sh +mkdir bviorg; +cd bviorg; +wget https://data.bris.ac.uk/datasets/tar/3h0hduxrq4awq2ffvhabjzbzi1.zip +unzip 3h0hduxrq4awq2ffvhabjzbzi1.zip +cd .. +``` + +To create the tvdorg directory: +```sh +mkdir tvdorg; +# download TVD dataset from https://multimedia.tencent.com/resources/tvd +``` + +With this file structure, the ``paths.json`` is simply edited by just replacing ``/path/to`` by the absolute path of the root directory of the experiment. +If your datasets are on different storages, just edit the relevant lines in ``paths.json``. + +### Creation of the consolidated configuration file +A unique configuration is create by mergind all the different files using the following command: +```sh +python3 tools/create_config.py src/training_scripts/NN_Filtering_HOP/default_config.json src/training_scripts/NN_Filtering_HOP/model/model.json src/training_scripts/NN_Filtering_HOP/paths.json > my_config.json +``` + +Then this file will be used in command line below, you should be able to run the process just by copy/pasting all lines of shell below. + Other keys should not be edited except for testing reasons. +### Advanced customization +The folllowing keys can be edited to adapt to your needs without affecting the results: +``` +/verbose +/stageX/training/dataloader/num_workers +/stageX/training/loggers/* +``` + ## I- Model Stage I Total size required for stage1 (without deleting intermediate data) is about 3312GB. @@ -18,8 +89,8 @@ Total size required for stage1 (without deleting intermediate data) is about 331 Convert div2k (4:4:4 RGB -> YUV420 10 bits): ```sh - python3 tools/convert_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset dataset/div2k_train --augmentation --output_location stage1/yuv - python3 tools/convert_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset dataset/div2k_valid --output_location stage1/yuv_valid + python3 tools/convert_dataset.py --json_config my_config.json --input_dataset dataset/div2k_train --augmentation --output_location stage1/yuv + python3 tools/convert_dataset.py --json_config my_config.json --input_dataset dataset/div2k_valid --output_location stage1/yuv_valid ``` dataset files are placed in the target directory (as set in the config.json ["stage1"]["yuv"]["path"]), a json file named ["stage1"]["yuv"]["dataset_filename"] is updated with the new data. @@ -40,8 +111,8 @@ Other macros can be set to 0. Extract cfg files and encoding/decoding script: ```sh - python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage1/yuv --output_location stage1/encdec - python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage1/yuv_valid --output_location stage1/encdec_valid + python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage1/yuv --output_location stage1/encdec + python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage1/yuv_valid --output_location stage1/encdec_valid ``` It will generate the cfg files for the dataset and a shell script to encode and decode all sequences in the dataset in the directory ["stage1"]["encdec"]["path"]. @@ -70,8 +141,8 @@ The script will perform the following step for each sequence: #### 4. Create a consolidated dataset ```sh -python3 tools/concatenate_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dir_json stage1/encdec --output_json stage1/encdec -python3 tools/concatenate_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dir_json stage1/encdec_valid --output_json stage1/encdec_valid +python3 tools/concatenate_dataset.py --json_config my_config.json --input_dir_json stage1/encdec --output_json stage1/encdec +python3 tools/concatenate_dataset.py --json_config my_config.json --input_dir_json stage1/encdec_valid --output_json stage1/encdec_valid ``` It will generate a unique dataset in ["stage1"]["encdec"]["path"] from all individual datasets in ["stage1"]["encdec"]["path"]/["dump_dir"] and encoder logs in ["stage1"]["encdec"]["enc_dir"]. @@ -79,11 +150,11 @@ It will generate a unique dataset in ["stage1"]["encdec"]["path"] from all indiv #### 5a. Create an offline dataset with all batches ```sh -python3 tools/create_unified_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json \ +python3 tools/create_unified_dataset.py --json_config my_config.json \ --nb_patches -1 --patch_size 128 --border_size 8 --input_dataset stage1/encdec \ --components org_Y,org_U,org_V,pred_Y,pred_U,pred_V,rec_before_dbf_Y,rec_before_dbf_U,rec_before_dbf_V,bs_Y,bs_U,bs_V,qp_base,qp_slice,ipb_Y \ --output_location stage1/dataset -python3 tools/create_unified_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json \ +python3 tools/create_unified_dataset.py --json_config my_config.json \ --nb_patches -1 --patch_size 128 --border_size 8 --input_dataset stage1/encdec_valid \ --components org_Y,org_U,org_V,pred_Y,pred_U,pred_V,rec_before_dbf_Y,rec_before_dbf_U,rec_before_dbf_V,bs_Y,bs_U,bs_V,qp_base,qp_slice,ipb_Y \ --output_location stage1/dataset_valid @@ -109,11 +180,7 @@ If you need to adapt the settings of your device for training, please edit the f When ready, simply run: ```sh -python3 training_scripts/NN_Filtering_HOP/training/main.py \ - training_scripts/NN_Filtering_HOP/config.json \ - training_scripts/NN_Filtering_HOP/model/model.json \ - training_scripts/NN_Filtering_HOP/training/cfg/training_default.json \ - training_scripts/NN_Filtering_HOP/training/cfg/stage1.json +python3 training_scripts/NN_Filtering_HOP/training/main.py --json_config my_config.json --stage 1 ``` ** Note ** The training is expected to be about 7h per epoch on a A100. @@ -121,7 +188,7 @@ python3 training_scripts/NN_Filtering_HOP/training/main.py \ The last ONNX model is converted into float SADL format. ```sh python3 training_scripts/NN_Filtering_HOP/convert/to_sadl.py \ - --json_config training_scripts/NN_Filtering_HOP/config.json \ + --json_config my_config.json \ --input_model stage1/training --output_model stage1/conversion ``` @@ -190,10 +257,10 @@ To do so, the model ``HOP_float.I.sadl`` is used to generate the dataset for sta Convert BVI-DVC 64 frames sequence into 65 frames sequence, and create the TVD dataset file (YUV are still taken from source). ```sh - python3 tools/convert_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset dataset/bvi --output_location stage2/yuv_bvi --extend 65 - python3 tools/convert_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset dataset/bvi_valid --output_location stage2/yuv_bvi_valid --extend 65 - python3 tools/convert_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset dataset/tvd --output_location stage2/yuv_tvd - python3 tools/convert_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset dataset/tvd_valid --output_location stage2/yuv_tvd_valid + python3 tools/convert_dataset.py --json_config my_config.json --input_dataset dataset/bvi --output_location stage2/yuv_bvi --extend 65 + python3 tools/convert_dataset.py --json_config my_config.json --input_dataset dataset/bvi_valid --output_location stage2/yuv_bvi_valid --extend 65 + python3 tools/convert_dataset.py --json_config my_config.json --input_dataset dataset/tvd --output_location stage2/yuv_tvd + python3 tools/convert_dataset.py --json_config my_config.json --input_dataset dataset/tvd_valid --output_location stage2/yuv_tvd_valid ``` dataset files are placed in the target directory (as set in the config.json ["stage2"]["yuv"]["path"]), a json file named ["stage2"]["yuv"]["dataset_filename"] is updated with the new data. @@ -214,10 +281,10 @@ Other macros can be set to 0. Extract cfg files and encoding/decoding script: ```sh -python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage2/yuv_tvd --output_location stage2/encdec_tvd -python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage2/yuv_tvd_valid --output_location stage2/encdec_tvd_valid -python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage2/yuv_bvi --output_location stage2/encdec_bvi -python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage2/yuv_bvi_valid --output_location stage2/encdec_bvi_valid +python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage2/yuv_tvd --output_location stage2/encdec_tvd +python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage2/yuv_tvd_valid --output_location stage2/encdec_tvd_valid +python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage2/yuv_bvi --output_location stage2/encdec_bvi +python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage2/yuv_bvi_valid --output_location stage2/encdec_bvi_valid ``` It will generate the cfg files for the dataset and a shell script to encode and decode all sequences in the dataset in the directory ["stage2"]["encdec"]["path"]. @@ -248,7 +315,7 @@ or you can use the script to encode on your cluster. N is the number of sequence ```sh LIST="bvi bvi_valid tvd tvd_valid"; for DS in $LIST; do - python3 tools/concatenate_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dir_json stage2/encdec_${DS} --input_dir_dataset_json stage2/yuv_${DS} --output_json stage2/encdec_${DS} + python3 tools/concatenate_dataset.py --json_config my_config.json --input_dir_json stage2/encdec_${DS} --input_dir_dataset_json stage2/yuv_${DS} --output_json stage2/encdec_${DS} done ``` It will generate a unique dataset for each dataset in ["stage2"]["encdec"]["path"] from all individual datasets in ["stage2"]["encdec_xxx"]["path"]/["dump_dir"] and encoder logs in ["stage2"]["encdec_xxx"]["enc_dir"]. @@ -260,12 +327,12 @@ It will generate a unique dataset for each dataset in ["stage2"]["encdec"]["path ```sh LIST="bvi tvd"; for DS in $LIST; do - python3 tools/create_unified_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json \ + python3 tools/create_unified_dataset.py --json_config my_config.json \ --nb_patches -1 --patch_size 128 --border_size 8 --input_dataset stage2/encdec_${DS} \ --components org_Y,org_U,org_V,pred_Y,pred_U,pred_V,rec_before_dbf_Y,rec_before_dbf_U,rec_before_dbf_V,bs_Y,bs_U,bs_V,qp_base,qp_slice,ipb_Y \ --output_location stage2/dataset/dataset_files/${DS} - python3 tools/create_unified_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json \ + python3 tools/create_unified_dataset.py --json_config my_config.json \ --nb_patches -1 --patch_size 128 --border_size 8 --input_dataset stage2/encdec_${DS}_valid \ --components org_Y,org_U,org_V,pred_Y,pred_U,pred_V,rec_before_dbf_Y,rec_before_dbf_U,rec_before_dbf_V,bs_Y,bs_U,bs_V,qp_base,qp_slice,ipb_Y \ --output_location stage2/dataset_valid/dataset_files/${DS} @@ -288,16 +355,13 @@ If you need to adapt the settings of your device for training, please edit the f When ready, simply run: ```sh -python3 training_scripts/NN_Filtering_HOP/training/main.py training_scripts/NN_Filtering_HOP/config.json \ - training_scripts/NN_Filtering_HOP/model/model.json \ - training_scripts/NN_Filtering_HOP/training/cfg/training_default.json \ - training_scripts/NN_Filtering_HOP/training/cfg/stage2.json +python3 training_scripts/NN_Filtering_HOP/training/main.py --json_config my_config.json --stage 2 ``` #### 2. Convert model The last ONNX model is converted into float SADL format. ```sh -python3 training_scripts/NN_Filtering_HOP/convert/to_sadl.py --json_config training_scripts/NN_Filtering_HOP/config.json \ +python3 training_scripts/NN_Filtering_HOP/convert/to_sadl.py --json_config my_config.json \ --input_model stage2/training --output_model stage2/conversion ``` @@ -367,10 +431,10 @@ Other macros can be set to 0. Extract cfg files and encoding/decoding script: ```sh -python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage2/yuv_tvd --output_location stage3/encdec_tvd -python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage2/yuv_tvd_valid --output_location stage3/encdec_tvd_valid -python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage2/yuv_bvi --output_location stage3/encdec_bvi -python3 tools/dataset_to_encoding_script.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dataset stage2/yuv_bvi_valid --output_location stage3/encdec_bvi_valid +python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage2/yuv_tvd --output_location stage3/encdec_tvd +python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage2/yuv_tvd_valid --output_location stage3/encdec_tvd_valid +python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage2/yuv_bvi --output_location stage3/encdec_bvi +python3 tools/dataset_to_encoding_script.py --json_config my_config.json --input_dataset stage2/yuv_bvi_valid --output_location stage3/encdec_bvi_valid ``` It will generate the cfg files for the dataset and a shell script to encode and decode all sequences in the dataset in the directory ["stage3"]["encdec"]["path"]. @@ -400,7 +464,7 @@ This step is similar to II-A-4. ```sh LIST="bvi bvi_valid tvd tvd_valid"; for DS in $LIST; do - python3 tools/concatenate_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json --input_dir_json stage3/encdec_${DS} --input_dir_dataset_json stage2/yuv_${DS} --output_json stage3/encdec_${DS} + python3 tools/concatenate_dataset.py --json_config my_config.json --input_dir_json stage3/encdec_${DS} --input_dir_dataset_json stage2/yuv_${DS} --output_json stage3/encdec_${DS} done ``` It will generate a unique dataset for each dataset in ["stage3"]["encdec"]["path"] from all individual datasets in ["stage3"]["encdec_xxx"]["path"]/["dump_dir"] and encoder logs in ["stage3"]["encdec_xxx"]["enc_dir"]. @@ -410,12 +474,12 @@ This step is similar to II-A-5a. ```sh LIST="bvi tvd"; for DS in $LIST; do - python3 tools/create_unified_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json \ + python3 tools/create_unified_dataset.py --json_config my_config.json \ --nb_patches -1 --patch_size 128 --border_size 8 --input_dataset stage3/encdec_${DS} \ --components org_Y,org_U,org_V,pred_Y,pred_U,pred_V,rec_before_dbf_Y,rec_before_dbf_U,rec_before_dbf_V,bs_Y,bs_U,bs_V,qp_base,qp_slice,ipb_Y \ --output_location stage3/dataset/dataset_files/${DS} - python3 tools/create_unified_dataset.py --json_config training_scripts/NN_Filtering_HOP/config.json \ + python3 tools/create_unified_dataset.py --json_config my_config.json \ --nb_patches -1 --patch_size 128 --border_size 8 --input_dataset stage3/encdec_${DS}_valid \ --components org_Y,org_U,org_V,pred_Y,pred_U,pred_V,rec_before_dbf_Y,rec_before_dbf_U,rec_before_dbf_V,bs_Y,bs_U,bs_V,qp_base,qp_slice,ipb_Y \ --output_location stage3/dataset_valid/dataset_files/${DS} @@ -432,16 +496,13 @@ If you need to adapt the settings of your device for training, please edit the f When ready, simply run: ```sh -python3 training_scripts/NN_Filtering_HOP/training/main.py --json_config training_scripts/NN_Filtering_HOP/config.json \ - training_scripts/NN_Filtering_HOP/model/model.json \ - training_scripts/NN_Filtering_HOP/training/cfg/training_default.json \ - training_scripts/NN_Filtering_HOP/training/cfg/stage3.json +python3 training_scripts/NN_Filtering_HOP/training/main.py --json_config my_config.json --stage 3 ``` #### 2. Convert model The last ONNX model is converted into float SADL format. ```sh -python3 training_scripts/NN_Filtering_HOP/convert/to_sadl.py --json_config training_scripts/NN_Filtering_HOP/config.json \ +python3 training_scripts/NN_Filtering_HOP/convert/to_sadl.py --json_config my_config.json \ --input_model stage3/training --output_model stage3/conversion ``` -- GitLab From 2e26dba6850c9c1247b0bed068098a8778ac17c0 Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Fri, 11 Aug 2023 14:50:02 +0200 Subject: [PATCH 09/12] remove unused key in config --- .../training_scripts/NN_Filtering_HOP/default_config.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/training/training_scripts/NN_Filtering_HOP/default_config.json b/training/training_scripts/NN_Filtering_HOP/default_config.json index b6c8c518a..b2ba595f2 100644 --- a/training/training_scripts/NN_Filtering_HOP/default_config.json +++ b/training/training_scripts/NN_Filtering_HOP/default_config.json @@ -617,11 +617,6 @@ "//": "path to output the model. input model is taken in training/ckpt_dir", "path": "/path/to/stage3/train", "model_filename": "model_float.sadl" - }, - "conversion_int": { - "//": "path to output the model. input model is taken in training/ckpt_dir", - "path": "/path/to/stage3/train", - "model_filename": "model_int16.sadl" } } } -- GitLab From 869d30df712a1511b5b59d8de1f6ba00b81b1184 Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Fri, 11 Aug 2023 15:03:05 +0200 Subject: [PATCH 10/12] rename deprecated model.json file (now included in the default_config.json) --- .../NN_Filtering_HOP/default_config.json | 43 +++++++++++++++++++ .../model/{model.json => model_sample.json} | 0 .../NN_Filtering_HOP/readme.md | 5 ++- 3 files changed, 47 insertions(+), 1 deletion(-) rename training/training_scripts/NN_Filtering_HOP/model/{model.json => model_sample.json} (100%) diff --git a/training/training_scripts/NN_Filtering_HOP/default_config.json b/training/training_scripts/NN_Filtering_HOP/default_config.json index b2ba595f2..f664cecc3 100644 --- a/training/training_scripts/NN_Filtering_HOP/default_config.json +++ b/training/training_scripts/NN_Filtering_HOP/default_config.json @@ -38,6 +38,49 @@ } }, + "model" : { + "class" : "Net", + "input_channels" : [ + [ + "rec_before_dbf_Y", + "rec_before_dbf_U", + "rec_before_dbf_V" + ], + [ + "pred_Y", + "pred_U", + "pred_V" + ], + [ + "bs_Y", + "bs_U", + "bs_V" + ], + [ "qp_base" ], + [ "qp_slice" ], + [ "ipb_Y" ] + ], + "input_kernels" : [ + 3, + 3, + 3, + 1, + 1, + 3 + ], + "D1" : 192, + "D2" : 32, + "D3" : 16, + "D4" : 16, + "D5" : 16, + "D6" : 48, + "N" : 24, + "C" : 64, + "C1" : 160, + "C21" : 32, + "C22" : 32, + "C31" : 64 + }, "stage1": { diff --git a/training/training_scripts/NN_Filtering_HOP/model/model.json b/training/training_scripts/NN_Filtering_HOP/model/model_sample.json similarity index 100% rename from training/training_scripts/NN_Filtering_HOP/model/model.json rename to training/training_scripts/NN_Filtering_HOP/model/model_sample.json diff --git a/training/training_scripts/NN_Filtering_HOP/readme.md b/training/training_scripts/NN_Filtering_HOP/readme.md index de9d12918..42d30f1e3 100644 --- a/training/training_scripts/NN_Filtering_HOP/readme.md +++ b/training/training_scripts/NN_Filtering_HOP/readme.md @@ -58,11 +58,14 @@ mkdir tvdorg; # download TVD dataset from https://multimedia.tencent.com/resources/tvd ``` + With this file structure, the ``paths.json`` is simply edited by just replacing ``/path/to`` by the absolute path of the root directory of the experiment. If your datasets are on different storages, just edit the relevant lines in ``paths.json``. ### Creation of the consolidated configuration file -A unique configuration is create by mergind all the different files using the following command: +The main configuration file is ``default_config.json`` and will be merged with the ``paths.json`` file to create a unique configuration file. Note that the model is also embedded in the default configuration file. + +A unique configuration is create by merging all the different files using the following command: ```sh python3 tools/create_config.py src/training_scripts/NN_Filtering_HOP/default_config.json src/training_scripts/NN_Filtering_HOP/model/model.json src/training_scripts/NN_Filtering_HOP/paths.json > my_config.json ``` -- GitLab From 9bc166bb1af6d8a28a44cb7759c06cf544150024 Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Sat, 12 Aug 2023 06:50:42 +0000 Subject: [PATCH 11/12] fix key name --- training/training_scripts/NN_Filtering_HOP/training/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/training_scripts/NN_Filtering_HOP/training/trainer.py b/training/training_scripts/NN_Filtering_HOP/training/trainer.py index 070b471f0..2143fe0a4 100644 --- a/training/training_scripts/NN_Filtering_HOP/training/trainer.py +++ b/training/training_scripts/NN_Filtering_HOP/training/trainer.py @@ -58,7 +58,7 @@ class Trainer: config: config hyperparameters stored in dictionary. See cfg/base_desc.txt for more details. """ self.config = config - self.config_stage = f"stage{stage}" + self.config_stage = config[f"stage{stage}"] self.config_training = self.config_stage["training"] self.current_epoch = 0 self.current_iteration = 0 -- GitLab From 4e14c01edcefb7076701b6cb459c36e5a2d15aed Mon Sep 17 00:00:00 2001 From: Franck Galpin <franck.galpin@interdigital.com> Date: Sat, 12 Aug 2023 09:36:51 +0200 Subject: [PATCH 12/12] corrections json and readme --- .../NN_Filtering_HOP/default_config.json | 53 ++++--------------- .../model/{model_sample.json => model.json} | 0 .../NN_Filtering_HOP/paths.json | 16 +++--- .../NN_Filtering_HOP/readme.md | 9 ++-- 4 files changed, 20 insertions(+), 58 deletions(-) rename training/training_scripts/NN_Filtering_HOP/model/{model_sample.json => model.json} (100%) diff --git a/training/training_scripts/NN_Filtering_HOP/default_config.json b/training/training_scripts/NN_Filtering_HOP/default_config.json index f664cecc3..a1820e611 100644 --- a/training/training_scripts/NN_Filtering_HOP/default_config.json +++ b/training/training_scripts/NN_Filtering_HOP/default_config.json @@ -38,50 +38,6 @@ } }, - "model" : { - "class" : "Net", - "input_channels" : [ - [ - "rec_before_dbf_Y", - "rec_before_dbf_U", - "rec_before_dbf_V" - ], - [ - "pred_Y", - "pred_U", - "pred_V" - ], - [ - "bs_Y", - "bs_U", - "bs_V" - ], - [ "qp_base" ], - [ "qp_slice" ], - [ "ipb_Y" ] - ], - "input_kernels" : [ - 3, - 3, - 3, - 1, - 1, - 3 - ], - "D1" : 192, - "D2" : 32, - "D3" : 16, - "D4" : 16, - "D5" : 16, - "D6" : 48, - "N" : 24, - "C" : 64, - "C1" : 160, - "C21" : 32, - "C22" : 32, - "C31" : 64 - }, - "stage1": { "yuv": { @@ -134,6 +90,9 @@ }, "training": { "path": "/path/to/stage1/train", + "ckpt_dir": "ckpt", + "ckpt_reload": "last.ckpt", + "model_onnx_filename": "last.onnx", "seed": 42, "//": "additional loggers (e.g. for tensorboard) can be implemented in logger.py and added here", "device": "cuda", @@ -362,6 +321,9 @@ }, "training": { "path": "/path/to/stage2/train", + "ckpt_dir": "ckpt", + "ckpt_reload": "last.ckpt", + "model_onnx_filename": "last.onnx", "seed": 42, "//": "additional loggers (e.g. for tensorboard) can be implemented in logger.py and added here", "device": "cuda", @@ -570,6 +532,9 @@ }, "training": { "path": "/path/to/stage3/train", + "ckpt_dir": "ckpt", + "ckpt_reload": "last.ckpt", + "model_onnx_filename": "last.onnx", "seed": 42, "//": "additional loggers (e.g. for tensorboard) can be implemented in logger.py and added here", "device": "cuda", diff --git a/training/training_scripts/NN_Filtering_HOP/model/model_sample.json b/training/training_scripts/NN_Filtering_HOP/model/model.json similarity index 100% rename from training/training_scripts/NN_Filtering_HOP/model/model_sample.json rename to training/training_scripts/NN_Filtering_HOP/model/model.json diff --git a/training/training_scripts/NN_Filtering_HOP/paths.json b/training/training_scripts/NN_Filtering_HOP/paths.json index 7dc8990bb..8ff3e94c3 100644 --- a/training/training_scripts/NN_Filtering_HOP/paths.json +++ b/training/training_scripts/NN_Filtering_HOP/paths.json @@ -122,7 +122,7 @@ }, "dataset": { "//": "path to store the full dataset which will be used by the training", - "path": "/path/to/stage2/dataset", + "path": "/path/to/stage2/dataset" }, "dataset_valid": { "//": "path to store the full dataset which will be used by the training", @@ -144,28 +144,28 @@ "path": "/path/to/stage3/encdec", "vtm_enc": "/path/to/src/bin/EncoderAppStatic", "vtm_dec": "/path/to/src/bin/DecoderAppStatic", - "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg" }, "encdec_bvi_valid": { "//": "path to store the shell script and all the generated files by the encoder/decoder", "path": "/path/to/stage3/encdec", "vtm_enc": "/path/to/src/bin/EncoderAppStatic", "vtm_dec": "/path/to/src/bin/DecoderAppStatic", - "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg" }, "encdec_tvd": { "//": "path to store the shell script and all the generated files by the encoder/decoder", "path": "/path/to/stage3/encdec", "vtm_enc": "/path/to/src/bin/EncoderAppStatic", "vtm_dec": "/path/to/src/bin/DecoderAppStatic", - "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg" }, "encdec_tvd_valid": { "//": "path to store the shell script and all the generated files by the encoder/decoder", "path": "/path/to/stage3/encdec", "vtm_enc": "/path/to/src/bin/EncoderAppStatic", "vtm_dec": "/path/to/src/bin/DecoderAppStatic", - "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg", + "vtm_cfg": "/path/to/src/cfg/encoder_randomaccess_vtm.cfg" }, "dataset": { "//": "path to store the full dataset which will be used by the training", @@ -180,11 +180,7 @@ }, "conversion": { "//": "path to output the model. input model is taken in training/ckpt_dir", - "path": "/path/to/stage3/train", - }, - "conversion_int": { - "//": "path to output the model. input model is taken in training/ckpt_dir", - "path": "/path/to/stage3/train", + "path": "/path/to/stage3/train" } } diff --git a/training/training_scripts/NN_Filtering_HOP/readme.md b/training/training_scripts/NN_Filtering_HOP/readme.md index 42d30f1e3..689c05e58 100644 --- a/training/training_scripts/NN_Filtering_HOP/readme.md +++ b/training/training_scripts/NN_Filtering_HOP/readme.md @@ -2,7 +2,7 @@ ## Overview ### Preparation of the directory -At minima, paths should first be set. The file ``training/training_scripts/NN_Filtering_HOP/paths.json`` should be edited to match your environment. +At minima, paths should first be set. The file ``training/training_scripts/NN_Filtering_HOP/paths.json`` should be copy and edited to match your environment. All keys with the name ``path`` should be edited to fit your particular environement. Additionally, you should also edit the variable ``vtm_xx`` to point to the VTM binaries and configuration files, the ``sadl_path`` to point to the sadl repository. @@ -63,11 +63,12 @@ With this file structure, the ``paths.json`` is simply edited by just replacing If your datasets are on different storages, just edit the relevant lines in ``paths.json``. ### Creation of the consolidated configuration file -The main configuration file is ``default_config.json`` and will be merged with the ``paths.json`` file to create a unique configuration file. Note that the model is also embedded in the default configuration file. - +The main configuration file is ``default_config.json`` and will be merged with the ``paths.json`` file to create a unique configuration file. A unique configuration is create by merging all the different files using the following command: ```sh -python3 tools/create_config.py src/training_scripts/NN_Filtering_HOP/default_config.json src/training_scripts/NN_Filtering_HOP/model/model.json src/training_scripts/NN_Filtering_HOP/paths.json > my_config.json +cp src/training_scripts/NN_Filtering_HOP/paths.json . +# edit paths.json +python3 src/training_scripts/tools/create_config.py src/training_scripts/NN_Filtering_HOP/default_config.json src/training_scripts/NN_Filtering_HOP/model/model.json paths.json > my_config.json ``` Then this file will be used in command line below, you should be able to run the process just by copy/pasting all lines of shell below. -- GitLab