draw: add badrect() function to reject zero, negative size or orverly huge rectangles

not checking the rectangle dimensions causes integer overflows
and memory corruption. adding a new badrect() function that checks
for these cases.

sys/include/ape/draw.h

@@ -408,6 +408,7 @@ extern int		cmap2rgb(int);
 extern int		cmap2rgba(int);
 extern void		icossin(int, int*, int*);
 extern void		icossin2(int, int, int*, int*);
+extern int		badrect(Rectangle);
 
 /*
  * Graphics

sys/include/draw.h

@@ -401,6 +401,7 @@ extern int		cmap2rgb(int);
 extern int		cmap2rgba(int);
 extern void		icossin(int, int*, int*);
 extern void		icossin2(int, int, int*, int*);
+extern int		badrect(Rectangle);
 
 /*
  * Graphics

sys/src/libdraw/alloc.c

@@ -26,6 +26,10 @@ _allocimage(Image *ai, Display *d, Rectangle r, ulong chan, int repl, ulong val,
 	err = 0;
 	i = 0;
 
+	if(badrect(r)){
+		werrstr("bad rectangle");
+		return nil;
+	}
 	if(chan == 0){
 		werrstr("bad channel descriptor");
 		return nil;

sys/src/libdraw/badrect.c

@@ -0,0 +1,22 @@
+#include <u.h>
+#include <libc.h>
+#include <draw.h>
+
+/*
+ * check for zero, negative size or insanely huge rectangle.
+ */
+int
+badrect(Rectangle r)
+{
+	int x, y;
+	uint z;
+
+	x = Dx(r);
+	y = Dy(r);
+	if(x > 0 && y > 0){
+		z = x*y;
+		if(z/x == y && z < 0x10000000)
+			return 0;
+	}
+	return 1;
+}

sys/src/libdraw/mkfile

@@ -6,6 +6,7 @@ OFILES=\
 	alloc.$O\
 	allocimagemix.$O\
 	arith.$O\
+	badrect.$O\
 	bezier.$O\
 	border.$O\
 	buildfont.$O\

sys/src/libmemdraw/alloc.c

@@ -27,14 +27,14 @@ allocmemimaged(Rectangle r, ulong chan, Memdata *md)
 	ulong l;
 	Memimage *i;
 
-	if(Dx(r) <= 0 || Dy(r) <= 0){
-		werrstr("bad rectangle %R", r);
-		return nil;
-	}
 	if((d = chantodepth(chan)) == 0) {
 		werrstr("bad channel descriptor %.8lux", chan);
 		return nil;
 	}
+	if(badrect(r)){
+		werrstr("bad rectangle %R", r);
+		return nil;
+	}
 
 	l = wordsperline(r, d);
 
@@ -76,8 +76,13 @@ allocmemimage(Rectangle r, ulong chan)
 		werrstr("bad channel descriptor %.8lux", chan);
 		return nil;
 	}
+	if(badrect(r)){
+		werrstr("bad rectangle %R", r);
+		return nil;
+	}
 
 	l = wordsperline(r, d);
+
 	nw = l*Dy(r);
 	md = malloc(sizeof(Memdata));
 	if(md == nil)

sys/src/libmemdraw/cload.c

@@ -9,7 +9,7 @@ cloadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	int y, bpl, c, cnt, offs;
 	uchar mem[NMEM], *memp, *omemp, *emem, *linep, *elinep, *u, *eu;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 	bpl = bytesperline(r, i->depth);
 	u = data;

sys/src/libmemdraw/draw.c

@@ -228,7 +228,7 @@ drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask,
 	int splitcoords;
 	Rectangle omr;
 
-	if(r->min.x>=r->max.x || r->min.y>=r->max.y)
+	if(badrect(*r))
 		return 0;
 	splitcoords = (p0->x!=p1->x) || (p0->y!=p1->y);
 	/* clip to destination */

sys/src/libmemdraw/load.c

@@ -10,7 +10,7 @@ loadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	Memdrawparam par;
 	uchar *q;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 
 	memset(&par, 0, sizeof par);

sys/src/libmemdraw/unload.c

@@ -9,7 +9,7 @@ unloadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	int y, l;
 	uchar *q;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 	l = bytesperline(r, i->depth);
 	if(ndata < l*Dy(r))