[Bug ipa/96502] New: attribute malloc effect lost after inlining

[msebor at gcc dot gnu.org]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96502

            Bug ID: 96502
           Summary: attribute malloc effect lost after inlining
           Product: gcc
           Version: 11.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: ipa
          Assignee: unassigned at gcc dot gnu.org
          Reporter: msebor at gcc dot gnu.org
                CC: marxin at gcc dot gnu.org
  Target Milestone: ---

The test case below demonstrates that the effect of the malloc function
attribute (as well as others) on subsequent optimizations is lost after a
function declared with it has been inlined.  This came up in the following
discussion:
https://gcc.gnu.org/pipermail/gcc-patches/2020-August/551526.html

$ cat x.c && gcc -O2 -S -Wall -Wextra -fdump-tree-optimized=/dev/stdout x.c
void* f (void);

__attribute__ ((malloc, noinline)) int* f0 (void) { return f (); }

void h0 (void)
{
  int *p = f0 ();
  int x = *p;
  int *q = f0 ();
  *q = 0;
  if (*p != x)             // folded to false
    __builtin_abort ();
}

__attribute__ ((malloc)) int* f1 (void) { return f (); }

void h1 (void)
{
  int *p = f1 ();
  int x = *p;
  int *q = f1 ();
  *q = 0;
  if (*p != x)             // not folded
    __builtin_abort ();
}


;; Function f0 (f0, funcdef_no=0, decl_uid=1933, cgraph_uid=1, symbol_order=0)

__attribute__((noinline, malloc))
f0 ()
{
  int * _3;

  <bb 2> [local count: 1073741824]:
  _3 = f (); [tail call]
  return _3;

}



;; Function h0 (h0, funcdef_no=1, decl_uid=1936, cgraph_uid=2, symbol_order=1)

h0 ()
{
  <bb 2> [local count: 1073741824]:
  f0 ();
  f0 (); [tail call]
  return;

}



;; Function f1 (f1, funcdef_no=2, decl_uid=1942, cgraph_uid=3, symbol_order=2)

__attribute__((malloc))
f1 ()
{
  int * _3;

  <bb 2> [local count: 1073741824]:
  _3 = f (); [tail call]
  return _3;

}



;; Function h1 (h1, funcdef_no=3, decl_uid=1945, cgraph_uid=4, symbol_order=3)

h1 ()
{
  int x;
  int _1;
  int * _6;
  int * _7;

  <bb 2> [local count: 1073741824]:
  _7 = f ();
  x_3 = *_7;
  _6 = f ();
  *_6 = 0;
  _1 = *_7;
  if (_1 != x_3)
    goto <bb 3>; [0.00%]
  else
    goto <bb 4>; [100.00%]

  <bb 3> [count: 0]:
  __builtin_abort ();

  <bb 4> [local count: 1073741824]:
  return;

}