Skip to content
Snippets Groups Projects
Commit ce6ee5e3 authored by Franck Galpin's avatar Franck Galpin
Browse files

solve memory error with xLOP model usiong block QP

parent b1715e6a
No related branches found
No related tags found
1 merge request!277solve memory error with xLOP model usiong block QP
......@@ -466,6 +466,78 @@ public:
}
}
#if JVET_AJ0124_QP_BLOCK
template<typename T>
static void fillInputFromBufTransSkipDCT(const Picture *pic, UnitArea inferArea, sadl::Tensor<T> &input, CPelUnitBuf buf,
bool luma, bool chroma, double scale, int shift)
{
assert(!chroma); // not tested
int hor, ver;
if (luma)
{
hor = inferArea.lwidth();
ver = inferArea.lheight();
}
else
{
hor = inferArea.lwidth() >> 1;
ver = inferArea.lheight() >> 1;
}
CPelBuf bufY, bufCb, bufCr;
if (luma)
{
bufY = buf.get(COMPONENT_Y);
}
if (chroma)
{
bufCb = buf.get(COMPONENT_Cb);
bufCr = buf.get(COMPONENT_Cr);
}
constexpr int height = 2; // dct size
constexpr int width = 2;
// DCT not needed for constant
if constexpr (std::is_same_v<T, float>)
{
for (int y = 0; y < ver; y += height)
{
for (int x = 0; x < hor; x += width)
{
if (luma)
{
input(0, y / height, x / width, 0) = bufY.at(x, y) /scale;
}
if (chroma)
{
input(0, y / height, x / width, 1) = bufCb.at(x/2, y/2) /scale;
input(0, y / height, x / width, 2) = bufCr.at(x/2, y/2) /scale;
}
}
}
}
else
{
for (int y = 0; y < ver; y += height)
{
for (int x = 0; x < hor; x += width)
{
if (luma)
{
input(0, y / height, x / width, 0) = bufY.at(x, y) << shift;
}
if (chroma)
{
input(0, y / height, x / width, 1) = bufCb.at(x/2, y/2) << shift;
input(0, y / height, x / width, 2) = bufCr.at(x/2, y/2) << shift;
}
}
}
}
}
#endif
template<typename T>
static void fillInputFromBufTrans(const Picture *pic, UnitArea inferArea, sadl::Tensor<T> &input, CPelUnitBuf buf,
bool luma, bool chroma, double scale, int shift)
......@@ -840,7 +912,7 @@ public:
break;
#if JVET_AJ0124_QP_BLOCK
case NN_INPUT_LOCAL_QP_BLOCK:
fillInputFromBufTrans<T>(pic, inferArea, inputs[inputData.index], pic->getBlockQpBuf(inferArea), inputData.luma, inputData.chroma, inputData.scale, inputData.shift);
fillInputFromBufTransSkipDCT<T>(pic, inferArea, inputs[inputData.index], pic->getBlockQpBuf(inferArea), inputData.luma, inputData.chroma, inputData.scale, inputData.shift);
break;
#endif
case NN_INPUT_SLICE_TYPE:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment