regexp out of mem

[Otto Moerbeek]

re http://securityreason.com/achievement_securityalert/102

This cherrypicks out of the netbsd fixes, without introducing
arbitray limits. 

This is enough to make their testcase succeed (I'll commit that one
soon).  I'm pretty sure some more changes are needed, though.
Especially some realloc expressions are suspect. 

Note that assert statements is not compiled in because of utils.h

        -Otto

Index: regcomp.c
===================================================================
RCS file: /cvs/src/lib/libc/regex/regcomp.c,v
retrieving revision 1.20
diff -u -p -r1.20 regcomp.c
--- regcomp.c   21 Nov 2010 00:02:30 -0000      1.20
+++ regcomp.c   5 Nov 2011 14:42:47 -0000
@@ -99,7 +99,7 @@ static sopno dupl(struct parse *, sopno,
 static void doemit(struct parse *, sop, size_t);
 static void doinsert(struct parse *, sop, size_t, sopno);
 static void dofwd(struct parse *, sopno, sop);
-static void enlarge(struct parse *, sopno);
+static int enlarge(struct parse *, sopno);
 static void stripsnug(struct parse *, struct re_guts *);
 static void findmust(struct parse *, struct re_guts *);
 static sopno pluscount(struct parse *, struct re_guts *);
@@ -1270,8 +1270,8 @@ dupl(struct parse *p,
        assert(finish >= start);
        if (len == 0)
                return(ret);
-       enlarge(p, p->ssize + len);     /* this many unexpected additions */
-       assert(p->ssize >= p->slen + len);
+       if (!enlarge(p, p->ssize + len)) /* this many unexpected additions */
+               return(ret);
        (void) memcpy((char *)(p->strip + p->slen),
                (char *)(p->strip + start), (size_t)len*sizeof(sop));
        p->slen += len;
@@ -1297,8 +1297,8 @@ doemit(struct parse *p, sop op, size_t o
 
        /* deal with undersized strip */
        if (p->slen >= p->ssize)
-               enlarge(p, (p->ssize+1) / 2 * 3);       /* +50% */
-       assert(p->slen < p->ssize);
+               if (!enlarge(p, (p->ssize+1) / 2 * 3))  /* +50% */
+                       return;
 
        /* finally, it's all reduced to the easy case */
        p->strip[p->slen++] = SOP(op, opnd);
@@ -1356,21 +1356,22 @@ dofwd(struct parse *p, sopno pos, sop va
 /*
  - enlarge - enlarge the strip
  */
-static void
+static int
 enlarge(struct parse *p, sopno size)
 {
        sop *sp;
 
        if (p->ssize >= size)
-               return;
+               return 1;
 
        sp = (sop *)realloc(p->strip, size*sizeof(sop));
        if (sp == NULL) {
                SETERROR(REG_ESPACE);
-               return;
+               return 0;
        }
        p->strip = sp;
        p->ssize = size;
+       return 1;
 }
 
 /*