[Bug c/78284] New: warn on malloc with very large arguments

[msebor at gcc dot gnu.org]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78284

            Bug ID: 78284
           Summary: warn on malloc with very large arguments
           Product: gcc
           Version: 7.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: c
          Assignee: unassigned at gcc dot gnu.org
          Reporter: msebor at gcc dot gnu.org
  Target Milestone: ---

Calling an allocation function such as malloc with an argument in excess of
SIZE_MAX / 2 can be assumed to fail because if it didn't, pointers to the
beginning and end of the large object could not be subtracted from one another
without overflowing the type of the subtraction (ptrdiff_t).  Passing a signed
argument to an allocation function that has a negative value (and so converts
to a very positive unsigned value) is a programming error that can subsequently
lead to buffer overflow.  GCC could help detect these kinds errors by checking
for the argument being definitely or potentially negative (e.g., in a nagative
range) and issuing warnings similar to those already issued by
-Walloca-larger-than.  In addition, for calloc, GCC could help detect the
overflow in the multiplication of the two size arguments (this enhancement is
the subject of bug 77531).

$ cat b.c && gcc -O2 -S -Wall -Wextra -fdump-tree-optimized=/dev/stdout b.c
typedef __SIZE_TYPE__ size_t;

void* f_alloca (void)
{
  int n = -1;
  return __builtin_alloca (n);
}

void* f_malloc (void)
{
  int n = -1;
  return __builtin_malloc (n);
}

void* f_realloc (void *p)
{
  int n = -1;
  return __builtin_realloc (p, n);
}

void* f_calloc (void)
{
  int m = 1;
  int n = -1;
  return __builtin_calloc (m, n);
}


;; Function f_alloca (f_alloca, funcdef_no=0, decl_uid=1796, cgraph_uid=0,
symbol_order=0)

f_alloca ()
{
  void * _3;

  <bb 2>:
  _3 = __builtin_alloca (18446744073709551615);
  return _3;

}



;; Function f_malloc (f_malloc, funcdef_no=1, decl_uid=1800, cgraph_uid=1,
symbol_order=1)

f_malloc ()
{
  void * _3;

  <bb 2>:
  _3 = __builtin_malloc (18446744073709551615); [tail call]
  return _3;

}



;; Function f_realloc (f_realloc, funcdef_no=2, decl_uid=1804, cgraph_uid=2,
symbol_order=2)

f_realloc (void * p)
{
  void * _4;

  <bb 2>:
  _4 = __builtin_realloc (p_2(D), 18446744073709551615); [tail call]
  return _4;

}



;; Function f_calloc (f_calloc, funcdef_no=3, decl_uid=1808, cgraph_uid=3,
symbol_order=3)

f_calloc ()
{
  void * _3;

  <bb 2>:
  _3 = __builtin_calloc (1, 18446744073709551615); [tail call]
  return _3;

}