[Bug middle-end/96838] New: missing warning on integer overflow in calls to allocation functions

[msebor at gcc dot gnu.org]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96838

            Bug ID: 96838
           Summary: missing warning on integer overflow in calls to
                    allocation functions
           Product: gcc
           Version: 11.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: middle-end
          Assignee: unassigned at gcc dot gnu.org
          Reporter: msebor at gcc dot gnu.org
  Target Milestone: ---

Passing large integer in excess of the range of the argument to an allocation
function results in integer overflow or wraparound, both of which can then lead
to a buffer overflow.  The problem isn't detected because the argument, after
conversion to the narrower type, has an unknown range.  The overflow/wraparound
could be detected if in this case the get_size_range() function also considered
the range of the argument before the conversion and indicated the problem to
the caller.

$ cat q.c && gcc -O2 -S -Wall -fdump-tree-optimized=/dev/stdout q.c
typedef __SIZE_TYPE__ size_t;

void* salloc (int) __attribute__ ((alloc_size (1)));
void* ualloc (unsigned) __attribute__ ((alloc_size (1)));

void* f (size_t n)
{
  if (n <= __INT_MAX__)
    n =  __INT_MAX__ + 1LU;

  void *p = salloc (n);          // missing warning: integer overflow
  __builtin_memset (p, 0, n);    // buffer overflow here
  return p;
}

void* g (size_t n)
{
  if (n <= __INT_MAX__)
    n =  __INT_MAX__ + 1LU;

  void *p = ualloc (n);          // missing warning: integer wraparound
  __builtin_memset (p, 0, n);    // buffer overflow here
  return p;
}


;; Function f (f, funcdef_no=0, decl_uid=1936, cgraph_uid=1, symbol_order=0)

f (size_t n)
{
  void * p;
  int _1;
  long unsigned int _3;

  <bb 2> [local count: 1073741824]:
  _3 = MAX_EXPR <n_2(D), 2147483648>;
  _1 = (int) _3;
  p_6 = salloc (_1);
  __builtin_memset (p_6, 0, _3);
  return p_6;

}



;; Function g (g, funcdef_no=1, decl_uid=1940, cgraph_uid=2, symbol_order=1)

g (size_t n)
{
  void * p;
  unsigned int _1;
  long unsigned int _3;

  <bb 2> [local count: 1073741824]:
  _3 = MAX_EXPR <n_2(D), 2147483648>;
  _1 = (unsigned int) _3;
  p_6 = ualloc (_1);
  __builtin_memset (p_6, 0, _3);
  return p_6;

}