This makes the rtvec_alloc argument size_t catching overflow and
truncated arguments (from "invalid" testcases), verifying the
argument against INT_MAX which is the limit set by the int
typed rtvec_def.num_elem member.
Bootstrapped and tested on x86_64-unknown-linux-gnu, pushed.
2021-05-12 Richard Biener <rguent...@suse.de>
PR middle-end/100547
* rtl.h (rtvec_alloc): Make argument size_t.
* rtl.c (rtvec_alloc): Verify the count is less than INT_MAX.
---
gcc/rtl.c | 5 ++++-
gcc/rtl.h | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/gcc/rtl.c b/gcc/rtl.c
index 035eadc3514..b0ba1ff684c 100644
--- a/gcc/rtl.c
+++ b/gcc/rtl.c
@@ -158,10 +158,13 @@ static size_t rtvec_alloc_sizes;
Store the length, and initialize all elements to zero. */
rtvec
-rtvec_alloc (int n)
+rtvec_alloc (size_t n)
{
rtvec rt;
+ /* rtvec_def.num_elem is an int. */
+ gcc_assert (n < INT_MAX);
+
rt = ggc_alloc_rtvec_sized (n);
/* Clear out the vector. */
memset (&rt->elem[0], 0, n * sizeof (rtx));
diff --git a/gcc/rtl.h b/gcc/rtl.h
index c5f3d20fae4..35178b5bfac 100644
--- a/gcc/rtl.h
+++ b/gcc/rtl.h
@@ -2976,7 +2976,7 @@ extern rtx rtx_alloc_stat_v (RTX_CODE MEM_STAT_DECL, int);
(sizeof (struct hwivec_def) \
+ ((NWORDS)-1) * sizeof (HOST_WIDE_INT))) \
-extern rtvec rtvec_alloc (int);
+extern rtvec rtvec_alloc (size_t);
extern rtvec shallow_copy_rtvec (rtvec);
extern bool shared_const_p (const_rtx);
extern rtx copy_rtx (rtx);
--
2.26.2
