Bug#838650: pixman: rowstride integer overflow

[Alessandro Vesely]

Source: pixman
Version: pixman-0.32.6
Severity: normal
Tags: upstream patch

Dear Maintainer,

it is wrong to compute offsets like so:

   int rowstride = something;
   char *buffer = base_ptr + y*rowstride + x*4;

That idiom fails in 64bit architecture where integers are 32 bit.  Consider a
not-so-uncommon A0 poster at 600 dpi.  It results in a 19860x28080 image.
While width and heights are 16 bit numbers, their product multiplied by a bpp
of 4 results in a negative integer.

Strides should be type size_t, or, if they can be negative, long integer.

The patch I attach just avoids crashes in various clients (inkscape, evince).
Package authors may want to carry out a clearer change.

Ale



-- System Information:
Debian Release: 8.6
  APT prefers stable
  APT policy: (500, 'stable')
Architecture: amd64 (x86_64)
Foreign Architectures: i386

Kernel: Linux 3.16.0-4-amd64 (SMP w/4 CPU cores)
Locale: LANG=en_US.utf8, LC_CTYPE=en_US.utf8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash
Init: systemd (via /run/systemd/system)

--- a/pixman/pixman-fast-path.c
+++ b/pixman/pixman-fast-path.c
@@ -2798,7 +2798,7 @@
 			    repeat (repeat_mode, &rx, bits->width);
 			    repeat (repeat_mode, &ry, bits->height);
 
-			    row = (uint8_t *)bits->bits + bits->rowstride * 4 * ry;
+			    row = (uint8_t *)bits->bits + (long)bits->rowstride * 4L * (long)ry;
 			    pixel = convert_pixel (row, rx) | mask;
 			}
 			else
@@ -2809,7 +2809,7 @@
 			    }
 			    else
 			    {
-				row = (uint8_t *)bits->bits + bits->rowstride * 4 * ry;
+				row = (uint8_t *)bits->bits + (long)bits->rowstride * 4L * (long)ry;
 				pixel = convert_pixel (row, rx) | mask;
 			    }
 			}
@@ -2911,8 +2911,8 @@
 	    repeat (repeat_mode, &x2, width);
 	    repeat (repeat_mode, &y2, height);
 
-	    row1 = (uint8_t *)bits->bits + bits->rowstride * 4 * y1;
-	    row2 = (uint8_t *)bits->bits + bits->rowstride * 4 * y2;
+	    row1 = (uint8_t *)bits->bits + (long)bits->rowstride * 4L * (long)y1;
+	    row2 = (uint8_t *)bits->bits + (long)bits->rowstride * 4L * (long)y2;
 
 	    tl = convert_pixel (row1, x1) | mask;
 	    tr = convert_pixel (row1, x2) | mask;
@@ -2947,7 +2947,7 @@
 	    }
 	    else
 	    {
-		row1 = (uint8_t *)bits->bits + bits->rowstride * 4 * y1;
+		row1 = (uint8_t *)bits->bits + (long)bits->rowstride * 4L * (long)y1;
 		row1 += bpp / 8 * x1;
 
 		mask1 = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
@@ -2960,7 +2960,7 @@
 	    }
 	    else
 	    {
-		row2 = (uint8_t *)bits->bits + bits->rowstride * 4 * y2;
+		row2 = (uint8_t *)bits->bits + (long)bits->rowstride * 4L * (long)y2;
 		row2 += bpp / 8 * x1;
 
 		mask2 = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
@@ -3058,7 +3058,7 @@
 		repeat (repeat_mode, &y0, height);
 	    }
 
-	    row = (uint8_t *)bits->bits + bits->rowstride * 4 * y0;
+	    row = (uint8_t *)bits->bits + (long)bits->rowstride * 4L * (long)y0;
 
 	    buffer[i] = convert_pixel (row, x0) | mask;
 	}