[PATCH] avoid infinite recursion in maybe_warn_alloc_args_overflow (pr 78775)

[Martin Sebor]

The attached patch avoids infinite recursion when traversing phi
nodes in maybe_warn_alloc_args_overflow by using a bitmap to keep
track of those already visited and breaking out.

Thanks
Martin

PR tree-optimization/78775 - ICE in maybe_warn_alloc_args_overflow

gcc/ChangeLog:

	PR tree-optimization/78775
	* calls.c (operand_signed_p): Add overload and avoid getting into
	infinite recursion when traversing phi nodes.

gcc/testsuite/ChangeLog:

	PR tree-optimization/78775
	* gcc.dg/pr78775.c: New test.

Index: gcc/calls.c
===================================================================
--- gcc/calls.c	(revision 243581)
+++ gcc/calls.c	(working copy)
@@ -1247,10 +1247,12 @@ alloc_max_size (void)
 }
 
 /* Return true if the type of OP is signed, looking through any casts
-   to an unsigned type.  */
+   to an unsigned type.  VISITED is expected to be initially null and
+   is used internally by recursive calls of the function.  Caller
+   must free *VISITED if non-null after the function returns.  */
 
 static bool
-operand_signed_p (tree op)
+operand_signed_p (tree op, bitmap *visited)
 {
   if (TREE_CODE (op) == SSA_NAME)
     {
@@ -1265,6 +1267,12 @@ static bool
 	}
       else if (gimple_code (def) == GIMPLE_PHI)
 	{
+	  if (!*visited)
+	    *visited = BITMAP_ALLOC (NULL);
+
+	  if (!bitmap_set_bit (*visited, SSA_NAME_VERSION (op)))
+	    return true;
+
 	  /* In a phi, a constant argument may be unsigned even
 	     if in the source it's signed and negative.  Ignore
 	     those and consider the result of a phi signed if
@@ -1274,7 +1282,7 @@ static bool
 	    {
 	      tree op = gimple_phi_arg_def (def, i);
 	      if (TREE_CODE (op) != INTEGER_CST
-		  && !operand_signed_p (op))
+		  && !operand_signed_p (op, visited))
 		return false;
 	    }
 
@@ -1285,6 +1293,21 @@ static bool
   return !TYPE_UNSIGNED (TREE_TYPE (op));
 }
 
+/* Return true if the type of OP is signed, looking through any casts
+   to an unsigned type.  */
+
+static bool
+operand_signed_p (tree op)
+{
+  bitmap visited = NULL;
+  bool ret = operand_signed_p (op, &visited);
+
+  if (visited)
+    BITMAP_FREE (visited);
+
+  return ret;
+}
+
 /* Diagnose a call EXP to function FN decorated with attribute alloc_size
    whose argument numbers given by IDX with values given by ARGS exceed
    the maximum object size or cause an unsigned oveflow (wrapping) when
Index: gcc/testsuite/gcc.dg/pr78775.c
===================================================================
--- gcc/testsuite/gcc.dg/pr78775.c	(revision 0)
+++ gcc/testsuite/gcc.dg/pr78775.c	(working copy)
@@ -0,0 +1,19 @@
+/* PR c/78775 - [7 Regression] ICE in maybe_warn_alloc_args_overflow
+   { dg-do compile }
+   { dg-options "-O2" } */
+
+int a, b, *c;
+
+int main (void)
+{
+  unsigned long d = 0;
+  while (1)
+    {
+      switch (b)
+      case 'S':
+	d = a;
+      c = __builtin_malloc (d);
+    }
+
+  return 0;
+}