[media] vivid: fix try_fmt behavior

vivid_try_fmt_vid_cap() called tpg_calc_line_width to calculate the sizeimage
value, but that tpg function uses the current format, not the proposed (tried)
format.

Rewrote this code to calculate this correctly.

The vivid_try_fmt_vid_out() code was completely wrong w.r.t. sizeimage, and
neither did it take the vdownsampling[] factors into account.

Signed-off-by: Hans Verkuil <hans.verkuil@cisco.com>
Tested-by: Vincent Abriou <vincent.abriou@st.com>
Signed-off-by: Mauro Carvalho Chehab <mchehab@s-opensource.com>

diff --git a/drivers/media/platform/vivid/vivid-vid-cap.c b/drivers/media/platform/vivid/vivid-vid-cap.c
index a18e6fec219bb5fe7fa381d6076fdccdd18bcc9e..01419455e545b4648469eb93d945b1959d3a90a9 100644 (file)
--- a/drivers/media/platform/vivid/vivid-vid-cap.c
+++ b/drivers/media/platform/vivid/vivid-vid-cap.c
@@ -616,7 +616,7 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv,
        /* This driver supports custom bytesperline values */
 
        mp->num_planes = fmt->buffers;
-       for (p = 0; p < mp->num_planes; p++) {
+       for (p = 0; p < fmt->buffers; p++) {
                /* Calculate the minimum supported bytesperline value */
                bytesperline = (mp->width * fmt->bit_depth[p]) >> 3;
                /* Calculate the maximum supported bytesperline value */
@@ -626,10 +626,17 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv,
                        pfmt[p].bytesperline = max_bpl;
                if (pfmt[p].bytesperline < bytesperline)
                        pfmt[p].bytesperline = bytesperline;
-               pfmt[p].sizeimage = tpg_calc_line_width(&dev->tpg, p, pfmt[p].bytesperline) *
-                       mp->height + fmt->data_offset[p];
+
+               pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) /
+                               fmt->vdownsampling[p] + fmt->data_offset[p];
+
                memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved));
        }
+       for (p = fmt->buffers; p < fmt->planes; p++)
+               pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height *
+                       (fmt->bit_depth[p] / fmt->vdownsampling[p])) /
+                       (fmt->bit_depth[0] / fmt->vdownsampling[0]);
+
        mp->colorspace = vivid_colorspace_cap(dev);
        if (fmt->color_enc == TGP_COLOR_ENC_HSV)
                mp->hsv_enc = vivid_hsv_enc_cap(dev);

diff --git a/drivers/media/platform/vivid/vivid-vid-out.c b/drivers/media/platform/vivid/vivid-vid-out.c
index 1a337306095452e493a0eb05d864557690fdc026..0b1b6218ede887601ffe7f5b277ebf2767f30d8a 100644 (file)
--- a/drivers/media/platform/vivid/vivid-vid-out.c
+++ b/drivers/media/platform/vivid/vivid-vid-out.c
@@ -390,22 +390,28 @@ int vivid_try_fmt_vid_out(struct file *file, void *priv,
 
        /* This driver supports custom bytesperline values */
 
-       /* Calculate the minimum supported bytesperline value */
-       bytesperline = (mp->width * fmt->bit_depth[0]) >> 3;
-       /* Calculate the maximum supported bytesperline value */
-       max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[0]) >> 3;
        mp->num_planes = fmt->buffers;
-       for (p = 0; p < mp->num_planes; p++) {
+       for (p = 0; p < fmt->buffers; p++) {
+               /* Calculate the minimum supported bytesperline value */
+               bytesperline = (mp->width * fmt->bit_depth[p]) >> 3;
+               /* Calculate the maximum supported bytesperline value */
+               max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[p]) >> 3;
+
                if (pfmt[p].bytesperline > max_bpl)
                        pfmt[p].bytesperline = max_bpl;
                if (pfmt[p].bytesperline < bytesperline)
                        pfmt[p].bytesperline = bytesperline;
-               pfmt[p].sizeimage = pfmt[p].bytesperline * mp->height;
+
+               pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) /
+                                       fmt->vdownsampling[p];
+
                memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved));
        }
        for (p = fmt->buffers; p < fmt->planes; p++)
-               pfmt[0].sizeimage += (pfmt[0].bytesperline * fmt->bit_depth[p]) /
-                                    (fmt->bit_depth[0] * fmt->vdownsampling[p]);
+               pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height *
+                       (fmt->bit_depth[p] / fmt->vdownsampling[p])) /
+                       (fmt->bit_depth[0] / fmt->vdownsampling[0]);
+
        mp->xfer_func = V4L2_XFER_FUNC_DEFAULT;
        mp->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
        mp->quantization = V4L2_QUANTIZATION_DEFAULT;