This patch, which brings in some bits of code from the master Go library, does a better job of detecting when a memory allocation request will overflow. This lets us panic in a way that the program can see, rather than crashing. Bootstrapped and ran Go testsuite on x86_64-unknown-linux-gnu. Committed to mainline and 4.7 branch.
Ian
diff -r 3e4478623419 libgo/runtime/chan.c --- a/libgo/runtime/chan.c Fri Sep 28 10:41:20 2012 -0700 +++ b/libgo/runtime/chan.c Fri Sep 28 14:06:03 2012 -0700 @@ -3,6 +3,8 @@ // license that can be found in the LICENSE file. #include "runtime.h" +#include "arch.h" +#include "malloc.h" #include "go-type.h" #define NOSELGEN 1 @@ -88,7 +90,7 @@ elem = t->__element_type; - if(hint < 0 || (int32)hint != hint || (elem->__size > 0 && (uintptr)hint > ((uintptr)-1) / elem->__size)) + if(hint < 0 || (int32)hint != hint || (elem->__size > 0 && (uintptr)hint > MaxMem / elem->__size)) runtime_panicstring("makechan: size out of range"); n = sizeof(*c); diff -r 3e4478623419 libgo/runtime/go-append.c --- a/libgo/runtime/go-append.c Fri Sep 28 10:41:20 2012 -0700 +++ b/libgo/runtime/go-append.c Fri Sep 28 14:06:03 2012 -0700 @@ -54,6 +54,9 @@ while (m < count); } + if ((uintptr) m > MaxMem / element_size) + runtime_panicstring ("growslice: cap out of range"); + n = __go_alloc (m * element_size); __builtin_memcpy (n, a.__values, a.__count * element_size); diff -r 3e4478623419 libgo/runtime/go-make-slice.c --- a/libgo/runtime/go-make-slice.c Fri Sep 28 10:41:20 2012 -0700 +++ b/libgo/runtime/go-make-slice.c Fri Sep 28 14:06:03 2012 -0700 @@ -37,7 +37,7 @@ if (cap < len || (uintptr_t) icap != cap || (std->__element_type->__size > 0 - && cap > (uintptr_t) -1U / std->__element_type->__size)) + && cap > MaxMem / std->__element_type->__size)) runtime_panicstring ("makeslice: cap out of range"); ret.__count = ilen; diff -r 3e4478623419 libgo/runtime/malloc.h --- a/libgo/runtime/malloc.h Fri Sep 28 10:41:20 2012 -0700 +++ b/libgo/runtime/malloc.h Fri Sep 28 14:06:03 2012 -0700 @@ -128,6 +128,15 @@ MaxGcproc = 4, }; +// Maximum memory allocation size, a hint for callers. +// This must be a #define instead of an enum because it +// is so large. +#if __SIZEOF_POINTER__ == 8 +#define MaxMem (16ULL<<30) /* 16 GB */ +#else +#define MaxMem ((uintptr)-1) +#endif + // A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).) struct MLink {