To correctly set the size of the image in a plane, it's needed
to divide the height of image by the vertical down sampling factor.
This was only happening in vivid_try_fmt_vid_cap(), but now it
applied in others sizeimage calculations as well.
Signed-off-by: André Almeida <andre.almeida@collabora.com>
Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
}
} else {
for (p = 0; p < buffers; p++)
}
} else {
for (p = 0; p < buffers; p++)
- sizes[p] = tpg_g_line_width(&dev->tpg, p) * h +
+ sizes[p] = (tpg_g_line_width(&dev->tpg, p) * h) /
+ dev->fmt_cap->vdownsampling[p] +
dev->fmt_cap->data_offset[p];
}
dev->fmt_cap->data_offset[p];
}
return -EINVAL;
}
for (p = 0; p < buffers; p++) {
return -EINVAL;
}
for (p = 0; p < buffers; p++) {
- size = tpg_g_line_width(&dev->tpg, p) * dev->fmt_cap_rect.height +
+ size = (tpg_g_line_width(&dev->tpg, p) *
+ dev->fmt_cap_rect.height) /
+ dev->fmt_cap->vdownsampling[p] +
dev->fmt_cap->data_offset[p];
if (vb2_plane_size(vb, p) < size) {
dev->fmt_cap->data_offset[p];
if (vb2_plane_size(vb, p) < size) {
for (p = 0; p < mp->num_planes; p++) {
mp->plane_fmt[p].bytesperline = tpg_g_bytesperline(&dev->tpg, p);
mp->plane_fmt[p].sizeimage =
for (p = 0; p < mp->num_planes; p++) {
mp->plane_fmt[p].bytesperline = tpg_g_bytesperline(&dev->tpg, p);
mp->plane_fmt[p].sizeimage =
- tpg_g_line_width(&dev->tpg, p) * mp->height +
+ (tpg_g_line_width(&dev->tpg, p) * mp->height) /
+ dev->fmt_cap->vdownsampling[p] +
dev->fmt_cap->data_offset[p];
}
return 0;
dev->fmt_cap->data_offset[p];
}
return 0;