[PATCH] vivid: fix vid_out_buf_prepare()

[Hans Verkuil]

The wrong size check was performed for output formats like NV24 which
set vfmt->buffers to 1, but vfmt->planes is 2. It was incorrectly
checking the payload size for plane 1, which doesn't exist.

Note: vfmt->buffers refers to the number of per-plane-buffers that
should be allocated. vfmt->planes refers to the number of planes
that make up an image. vfmt->planes may be > vfmt->buffers.

Signed-off-by: Hans Verkuil <hverkuil-ci...@xs4all.nl>
---
diff --git a/drivers/media/platform/vivid/vivid-vid-out.c 
b/drivers/media/platform/vivid/vivid-vid-out.c
index e45753a1adde..54e8fb23e336 100644
--- a/drivers/media/platform/vivid/vivid-vid-out.c
+++ b/drivers/media/platform/vivid/vivid-vid-out.c
@@ -99,17 +99,20 @@ static int vid_out_buf_out_validate(struct vb2_buffer *vb)
 static int vid_out_buf_prepare(struct vb2_buffer *vb)
 {
        struct vivid_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
-       unsigned long size;
-       unsigned planes;
+       const struct vivid_fmt *vfmt = dev->fmt_out;
+       unsigned planes = vfmt->buffers;
+       unsigned h = dev->fmt_out_rect.height;
+       unsigned size = dev->bytesperline_out[0] * h;
        unsigned p;

+       for (p = vfmt->buffers; p < vfmt->planes; p++)
+               size += dev->bytesperline_out[p] * h / vfmt->vdownsampling[p];
+
        dprintk(dev, 1, "%s\n", __func__);

        if (WARN_ON(NULL == dev->fmt_out))
                return -EINVAL;

-       planes = dev->fmt_out->planes;
-
        if (dev->buf_prepare_error) {
                /*
                 * Error injection: test what happens if buf_prepare() returns
@@ -120,11 +123,12 @@ static int vid_out_buf_prepare(struct vb2_buffer *vb)
        }

        for (p = 0; p < planes; p++) {
-               size = dev->bytesperline_out[p] * dev->fmt_out_rect.height +
-                       vb->planes[p].data_offset;
+               if (p)
+                       size = dev->bytesperline_out[p] * h;
+               size += vb->planes[p].data_offset;

                if (vb2_get_plane_payload(vb, p) < size) {
-                       dprintk(dev, 1, "%s the payload is too small for plane 
%u (%lu < %lu)\n",
+                       dprintk(dev, 1, "%s the payload is too small for plane 
%u (%lu < %u)\n",
                                        __func__, p, vb2_get_plane_payload(vb, 
p), size);
                        return -EINVAL;
                }