doc/software-manual.tex

@@ -5942,6 +5942,18 @@ Specifies sii_num_units_in_shutter_interval for single entry.If multiple entries
   \Default{0} &
   Specifies the bottom offset of spatial extrapolation for the \emph{i}-th neural network post-filter.
   \\
+  \Option{SEINNPFCAapplicationPurposeTagUriPresentFlag\emph{i}} &
+  \Default{false} &
+  Specifies the nnpfc_application_purpose_tag_uri_present_flag syntax element for the \emph{i}-th neural network post-filter.
+  \par
+    When true (non-zero) specifies the presence of the nnpfc_application_purpose_tag_uri syntax element in the NNPFC SEI message.
+  \par
+    When false specifies the absence of the nnpfc_application_purpose_tag_uri syntax element in the NNPFC SEI message.
+  \\
+  \Option{SEINNPFCApplicationPurposeTagUri\emph{i}} &
+  \Default{""} &
+  specifies a tag URI with syntax and semantics as specified in IETF RFC 4151 identifying the application determined purpose of the NNPF, when nnpfc_purpose is equal to 0 of the \emph{i}-th neural network post-filter.
+  \\
   \Option{SEINNPFCInputPicOutputFlag\emph{i}} &
   \Default{false} &
   Indicates whether the i-th neural network post filter generates a corresponding output picture for the i-th input picture. 

source/App/EncoderApp/EncApp.cpp

@@ -1281,6 +1281,13 @@ void EncApp::xInitLibCfg( int layerIdx )
         m_cEncLib.setNNPostFilterSEICharacteristicsSpatialExtrapolationRightOffset (m_nnPostFilterSEICharacteristicsSpatialExtrapolationRightOffset[i], i);
         m_cEncLib.setNNPostFilterSEICharacteristicsSpatialExtrapolationTopOffset   (m_nnPostFilterSEICharacteristicsSpatialExtrapolationTopOffset[i], i);
         m_cEncLib.setNNPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset(m_nnPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset[i], i);
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+      m_cEncLib.setNNPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag( m_nnPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag[i], i);
+      if (m_cEncLib.getNNPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag(i))
+      {
+        m_cEncLib.setNNPostFilterSEICharacteristicsSpatialExrapolationPrompt( m_nnPostFilterSEICharacteristicsSpatialExtrapolationPrompt[i], i);
+      }
+#endif
       }
 #endif
       m_cEncLib.setNNPostFilterSEICharacteristicsAbsentInputPicZeroFlag  (m_nnPostFilterSEICharacteristicsAbsentInputPicZeroFlag[i], i);

source/App/EncoderApp/EncAppCfg.cpp

@@ -2077,6 +2077,14 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] )
     std::ostringstream spatialExtrapolationBottomOffset;
     spatialExtrapolationBottomOffset << "SEINNPFCSpatialExtrapolationLeftOffset" << i; 
     opts.addOptions()(spatialExtrapolationBottomOffset.str(), m_nnPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset[i], 0, "Bottom offset of spatial extrapolation");
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+    std::ostringstream spatialextrapolationPromptPresentFlag;
+    spatialextrapolationPromptPresentFlag << "SEINNPFCSpatialExtrapolationPromptPresentFlag" << i;
+    opts.addOptions()(spatialextrapolationPromptPresentFlag.str(), m_nnPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag[i], false, "equal to 1 specifies that nnpfc_prompt syntax element is present and nnpfc_alignment_zero_bit_c syntax element may be present. nnpfc_spatial_extrapolation_prompt_present_flag equal to 0 specifies that nnpfc_prompt syntax element and nnpfc_alignment_zero_bit_c syntax element are not present.");
+    std::ostringstream spatialextrapolationPrompt;
+    spatialextrapolationPrompt << "SEINNPFCSpatialExtrapolationPrompt" << i;
+    opts.addOptions()(spatialextrapolationPrompt.str(), m_nnPostFilterSEICharacteristicsSpatialExtrapolationPrompt[i], std::string(""), "specifies the text string prompt used for generating the contents of the spatial extrapolation image area.");
+#endif
 #endif
     std::ostringstream InputPicOutputFlag;
     InputPicOutputFlag << "SEINNPFCInputPicOutputFlag" << i;

source/App/EncoderApp/EncAppCfg.h

@@ -829,6 +829,10 @@ protected:
   int32_t               m_nnPostFilterSEICharacteristicsSpatialExtrapolationRightOffset[MAX_NUM_NN_POST_FILTERS];
   int32_t               m_nnPostFilterSEICharacteristicsSpatialExtrapolationTopOffset[MAX_NUM_NN_POST_FILTERS];
   int32_t               m_nnPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset[MAX_NUM_NN_POST_FILTERS];
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+  bool                  m_nnPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag[MAX_NUM_NN_POST_FILTERS];
+  std::string           m_nnPostFilterSEICharacteristicsSpatialExtrapolationPrompt[MAX_NUM_NN_POST_FILTERS];
+#endif
 #endif
   std::vector<bool>     m_nnPostFilterSEICharacteristicsInputPicOutputFlag[MAX_NUM_NN_POST_FILTERS];
   bool                  m_nnPostFilterSEICharacteristicsAbsentInputPicZeroFlag[MAX_NUM_NN_POST_FILTERS];

source/Lib/CommonLib/SEI.cpp

@@ -1027,6 +1027,10 @@ SEINeuralNetworkPostFilterCharacteristics::SEINeuralNetworkPostFilterCharacteris
   m_spatialExtrapolationRightOffset = sei.m_spatialExtrapolationRightOffset;
   m_spatialExtrapolationTopOffset = sei.m_spatialExtrapolationTopOffset;
   m_spatialExtrapolationBottomOffset = sei.m_spatialExtrapolationBottomOffset;
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+  m_spatialExtrapolationPromptPresentFlag = sei.m_spatialExtrapolationPromptPresentFlag;
+  m_prompt =  sei.m_prompt;
+#endif
 #endif
   m_inputPicOutputFlag = sei.m_inputPicOutputFlag;
 }
@@ -1095,6 +1099,10 @@ bool SEINeuralNetworkPostFilterCharacteristics::operator == (const SEINeuralNetw
   m_spatialExtrapolationRightOffset == sei.m_spatialExtrapolationRightOffset &&
   m_spatialExtrapolationTopOffset == sei.m_spatialExtrapolationTopOffset &&
   m_spatialExtrapolationBottomOffset == sei.m_spatialExtrapolationBottomOffset &&
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+  m_spatialExtrapolationPromptPresentFlag == sei.m_spatialExtrapolationPromptPresentFlag  &&
+  m_prompt ==  sei.m_prompt  &&
+#endif
 #endif
   m_inputPicOutputFlag == sei.m_inputPicOutputFlag &&
   m_payloadLength == sei.m_payloadLength;

source/Lib/CommonLib/SEI.h

@@ -1440,6 +1440,10 @@ public:
     , m_spatialExtrapolationRightOffset(0)
     , m_spatialExtrapolationTopOffset(0)
     , m_spatialExtrapolationBottomOffset(0)
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+    , m_spatialExtrapolationPromptPresentFlag(false)
+    , m_prompt("")
+#endif
 #endif
     , m_absentInputPicZeroFlag(false)
     , m_numInpPicsInOutputTensor(0)
@@ -1527,6 +1531,10 @@ public:
   int32_t        m_spatialExtrapolationRightOffset;
   int32_t        m_spatialExtrapolationTopOffset;
   int32_t        m_spatialExtrapolationBottomOffset;
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+  bool          m_spatialExtrapolationPromptPresentFlag;
+  std::string   m_prompt;
+#endif
 #endif
   std::vector<bool> m_inputPicOutputFlag;
   bool           m_absentInputPicZeroFlag;

source/Lib/CommonLib/SEINeuralNetworkPostFiltering.cpp

@@ -186,6 +186,15 @@ void SEINeuralNetworkPostFiltering::filterPictures(PicList& picList)
     for (auto sei : currCodedPic->m_nnpfcActivated)
     {
       auto currNnpfc = (SEINeuralNetworkPostFilterCharacteristics*) sei;
+#if JVET_AI0061_PROPOSAL2_SPATIAL_EXTRAPOLATION
+      if ((currNnpfc->m_purpose & NNPC_PurposeType::SPATIAL_EXTRAPOLATION) != 0)
+      {
+        currNnpfc->m_overlap = 0;
+        currNnpfc->m_constantPatchSizeFlag = 1;
+        currNnpfc->m_patchWidthMinus1 = croppedWidth - 1;
+        currNnpfc->m_patchHeightMinus1 = croppedHeight - 1;
+      }
+#endif
       checkInputPics(currCodedPic, currNnpfc, sourceWidth, sourceHeight, croppedWidth, croppedHeight);
     }
 

source/Lib/CommonLib/TypeDef.h

@@ -70,6 +70,10 @@
 
 #define JVET_AI0153_OMI_SEI 1 // JVET-AI0153: OMI-SEI Implementation as JVET-AH0346
 
+#define JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1  1 //JVET-AI0061 proposal 1: signal a NNPFC prompt for spatial extrapolation
+
+#define JVET_AI0061_PROPOSAL2_SPATIAL_EXTRAPOLATION 1 // JVET AI0061-Proposal2: Ensure spatial extrapolation works correctly (Option 1)
+
 //########### place macros to be be kept below this line ###############
 
 #define GDR_ENABLED   1

source/Lib/DecoderLib/SEIread.cpp

@@ -3188,6 +3188,21 @@ void SEIReader::xParseSEINNPostFilterCharacteristics(SEINeuralNetworkPostFilterC
       sei.m_spatialExtrapolationTopOffset = value;
       sei_read_svlc(pDecodedMessageOutputStream, value, "nnpfc_spatial_extrapolation_bottom_offset");
       sei.m_spatialExtrapolationBottomOffset = value;
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+      sei_read_flag(pDecodedMessageOutputStream, val, "nnpfc_spatial_extrapolation_prompt_present_flag");
+      sei.m_spatialExtrapolationPromptPresentFlag = val;
+      if (sei.m_spatialExtrapolationPromptPresentFlag)
+      {
+        std::string valp;
+        while (!isByteAligned())
+        {
+          sei_read_flag(pDecodedMessageOutputStream, val, "nnpfc_alignment_zero_bit_c");
+          CHECK(val != 0, "nnpfc_alignment_zero_bit_c not equal to zero");
+        }
+        sei_read_string(pDecodedMessageOutputStream, valp, "nnpfc_prompt");
+        sei.m_prompt = valp;
+      }
+#endif
     }
 #endif
 
@@ -3306,28 +3321,35 @@ void SEIReader::xParseSEINNPostFilterCharacteristics(SEINeuralNetworkPostFilterC
       CHECK(sei.m_chromaSampleLocTypeFrame > Chroma420LocType::UNSPECIFIED, "The value of nnpfc_chroma_sample_loc_type_frame shall be in the range of 0 to 6, inclusive");
     }
 
-    sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_overlap");
-    sei.m_overlap = val;
+#if JVET_AI0061_PROPOSAL2_SPATIAL_EXTRAPOLATION
+    if((sei.m_purpose & NNPC_PurposeType::SPATIAL_EXTRAPOLATION) == 0)
+    {
+#endif
+      sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_overlap");
+      sei.m_overlap = val;
 
-    sei_read_flag(pDecodedMessageOutputStream, val, "nnpfc_constant_patch_size_flag");
-    sei.m_constantPatchSizeFlag = val;
+      sei_read_flag(pDecodedMessageOutputStream, val, "nnpfc_constant_patch_size_flag");
+      sei.m_constantPatchSizeFlag = val;
 
-    if (sei.m_constantPatchSizeFlag)
-    {
-      sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_patch_width_minus1");
-      sei.m_patchWidthMinus1 = val;
+      if (sei.m_constantPatchSizeFlag)
+      {
+        sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_patch_width_minus1");
+        sei.m_patchWidthMinus1 = val;
 
-      sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_patch_height_minus1");
-      sei.m_patchHeightMinus1 = val;
-    }
-    else
-    {
-      sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_extended_patch_width_cd_delta_minus1");
-      sei.m_extendedPatchWidthCdDeltaMinus1 = val;
+        sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_patch_height_minus1");
+        sei.m_patchHeightMinus1 = val;
+      }
+      else
+      {
+        sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_extended_patch_width_cd_delta_minus1");
+        sei.m_extendedPatchWidthCdDeltaMinus1 = val;
 
-      sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_extended_patch_height_cd_delta_minus1");
-      sei.m_extendedPatchHeightCdDeltaMinus1 = val;
+        sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_extended_patch_height_cd_delta_minus1");
+        sei.m_extendedPatchHeightCdDeltaMinus1 = val;
+      }
+#if JVET_AI0061_PROPOSAL2_SPATIAL_EXTRAPOLATION
     }
+#endif
 
     sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_padding_type");
     sei.m_paddingType = val;

source/Lib/EncoderLib/EncCfg.h

@@ -778,6 +778,10 @@ protected:
   int32_t                 m_nnPostFilterSEICharacteristicsSpatialExtrapolationRightOffset[MAX_NUM_NN_POST_FILTERS];
   int32_t                 m_nnPostFilterSEICharacteristicsSpatialExtrapolationTopOffset[MAX_NUM_NN_POST_FILTERS];
   int32_t                 m_nnPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset[MAX_NUM_NN_POST_FILTERS];
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+  bool                    m_nnPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag[MAX_NUM_NN_POST_FILTERS];
+  std::string             m_nnPostFilterSEICharacteristicsSpatialExtrapolationPrompt[MAX_NUM_NN_POST_FILTERS];
+#endif
 #endif
   std::vector<bool>       m_nnPostFilterSEICharacteristicsInputPicOutputFlag[MAX_NUM_NN_POST_FILTERS];
   bool                    m_nnPostFilterSEICharacteristicsAbsentInputPicZeroFlag[MAX_NUM_NN_POST_FILTERS];
@@ -2184,6 +2188,12 @@ public:
   int32_t     getNNPostFilterSEICharacteristicsSpatialExtrapolationTopOffset(int filterIdx)                             { return m_nnPostFilterSEICharacteristicsSpatialExtrapolationTopOffset[filterIdx]; }
   void        setNNPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset(int32_t value, int filterIdx)           { m_nnPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset[filterIdx] = value; }
   int32_t     getNNPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset(int filterIdx)                          { return m_nnPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset[filterIdx]; }
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+  void        setNNPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag(bool promptPresentFlag, int filterIdx) { m_nnPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag[filterIdx] = promptPresentFlag; }
+  bool        getNNPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag(int filterIdx) const                   { return m_nnPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag[filterIdx]; }
+  void        setNNPostFilterSEICharacteristicsSpatialExrapolationPrompt(std::string prompt, int filterIdx)                 { m_nnPostFilterSEICharacteristicsSpatialExtrapolationPrompt[filterIdx] = prompt; }
+  std::string getNNPostFilterSEICharacteristicsSpatialExrapolationPrompt(int filterIdx) const                               { return m_nnPostFilterSEICharacteristicsSpatialExtrapolationPrompt[filterIdx]; }
+#endif
 #endif
   void        setNNPostFilterSEICharacteristicsInputPicOutputFlag(std::vector<bool> value, int filterIdx)   { m_nnPostFilterSEICharacteristicsInputPicOutputFlag[filterIdx] = value; }
   const       std::vector<bool>& getNNPostFilterSEICharacteristicsInputPicOutputFlag(int filterIdx)         { return m_nnPostFilterSEICharacteristicsInputPicOutputFlag[filterIdx]; }

source/Lib/EncoderLib/SEIEncoder.cpp

@@ -1662,6 +1662,13 @@ void SEIEncoder::initSEINeuralNetworkPostFilterCharacteristics(SEINeuralNetworkP
       sei->m_spatialExtrapolationRightOffset = m_pcCfg->getNNPostFilterSEICharacteristicsSpatialExtrapolationRightOffset(filterIdx);
       sei->m_spatialExtrapolationTopOffset = m_pcCfg->getNNPostFilterSEICharacteristicsSpatialExtrapolationTopOffset(filterIdx);
       sei->m_spatialExtrapolationBottomOffset = m_pcCfg->getNNPostFilterSEICharacteristicsSpatialExtrapolationBottomOffset(filterIdx);
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+      sei->m_spatialExtrapolationPromptPresentFlag = m_pcCfg->getNNPostFilterSEICharacteristicsSpatialExtrapolationPromptPresentFlag(filterIdx);
+      if (sei->m_spatialExtrapolationPromptPresentFlag)
+      {
+        sei->m_prompt = m_pcCfg->getNNPostFilterSEICharacteristicsSpatialExrapolationPrompt(filterIdx);
+      }
+#endif
     }
 #endif
 

source/Lib/EncoderLib/SEIwrite.cpp

@@ -1933,6 +1933,17 @@ void SEIWriter::xWriteSEINeuralNetworkPostFilterCharacteristics(const SEINeuralN
       xWriteSvlc(sei.m_spatialExtrapolationRightOffset, "nnpfc_spatial_extrapolation_right_offset");
       xWriteSvlc(sei.m_spatialExtrapolationTopOffset, "nnpfc_spatial_extrapolation_top_offset");
       xWriteSvlc(sei.m_spatialExtrapolationBottomOffset, "nnpfc_spatial_extrapolation_right_offset");
+#if JVET_AI0061_SPATIAL_EXTRAPOLATION_PROPOSAL1
+      xWriteFlag(sei.m_spatialExtrapolationPromptPresentFlag, "nnpfc_spatial_extrapolation_prompt_present_flag");
+      if (sei.m_spatialExtrapolationPromptPresentFlag)
+      {
+        while (!isByteAligned())
+        {
+          xWriteFlag(0, "nnpfc_alignment_zero_bit_c");
+        }
+        xWriteString(sei.m_prompt, "nnpfc_prompt");
+      }
+#endif
     }
 #endif
 
@@ -1993,18 +2004,25 @@ void SEIWriter::xWriteSEINeuralNetworkPostFilterCharacteristics(const SEINeuralN
       xWriteUvlc(to_underlying(sei.m_chromaSampleLocTypeFrame), "nnpfc_chroma_sample_loc_type_frame");
     }
     
-    xWriteUvlc(sei.m_overlap, "nnpfc_overlap");
-    xWriteFlag(sei.m_constantPatchSizeFlag, "nnpfc_constant_patch_size_flag");
-    if (sei.m_constantPatchSizeFlag)
-    {
-    xWriteUvlc(sei.m_patchWidthMinus1, "nnpfc_patch_width_minus1");
-    xWriteUvlc(sei.m_patchHeightMinus1, "nnpfc_patch_height_minus1");
-    }
-    else
+#if JVET_AI0061_PROPOSAL2_SPATIAL_EXTRAPOLATION
+    if((sei.m_purpose & NNPC_PurposeType::SPATIAL_EXTRAPOLATION) == 0)
     {
-      xWriteUvlc(sei.m_extendedPatchWidthCdDeltaMinus1, "extended_nnpfc_patch_width_cd_delta_minus1");
-      xWriteUvlc(sei.m_extendedPatchHeightCdDeltaMinus1, "extended_nnpfc_patch_height_cd_delta_minus1");
+#endif
+      xWriteUvlc(sei.m_overlap, "nnpfc_overlap");
+      xWriteFlag(sei.m_constantPatchSizeFlag, "nnpfc_constant_patch_size_flag");
+      if (sei.m_constantPatchSizeFlag)
+      {
+        xWriteUvlc(sei.m_patchWidthMinus1, "nnpfc_patch_width_minus1");
+        xWriteUvlc(sei.m_patchHeightMinus1, "nnpfc_patch_height_minus1");
+      }
+      else
+      {
+        xWriteUvlc(sei.m_extendedPatchWidthCdDeltaMinus1, "extended_nnpfc_patch_width_cd_delta_minus1");
+        xWriteUvlc(sei.m_extendedPatchHeightCdDeltaMinus1, "extended_nnpfc_patch_height_cd_delta_minus1");
+      }
+#if JVET_AI0061_PROPOSAL2_SPATIAL_EXTRAPOLATION
     }
+#endif
     xWriteUvlc(sei.m_paddingType, "nnpfc_padding_type");
     if (sei.m_paddingType == NNPC_PaddingType::FIXED_PADDING)
     {

source/Lib/Utilities/VideoIOYuv.cpp

@@ -1478,8 +1478,10 @@ bool VideoIOYuv::writeUpscaledPicture(const SPS &sps, const PPS &pps, const CPel
     else
     {
       const Window &conf = pps.getConformanceWindow();
+      int curPicWidth = maxWidth   - SPS::getWinUnitX( sps.getChromaFormatIdc() ) * ( afterScaleWindowFullResolution.getWindowLeftOffset() + afterScaleWindowFullResolution.getWindowRightOffset() );
+      int curPicHeight = maxHeight - SPS::getWinUnitY( sps.getChromaFormatIdc() ) * ( afterScaleWindowFullResolution.getWindowTopOffset()  + afterScaleWindowFullResolution.getWindowBottomOffset() );
 
-      ret = write(maxWidth, maxHeight, pic, ipCSC,
+      ret = write(curPicWidth, curPicHeight, pic, ipCSC,
                   packedYuvOutputMode, conf.getWindowLeftOffset() * SPS::getWinUnitX(chromaFormatIdc),
                   conf.getWindowRightOffset() * SPS::getWinUnitX(chromaFormatIdc),
                   conf.getWindowTopOffset() * SPS::getWinUnitY(chromaFormatIdc),