Implement the OpenMP pinned memory trait on Linux hosts using the mlock
syscall. Pinned allocations are performed using mmap, not malloc, to ensure
that they can be unpinned safely when freed.
This implementation will work OK for page-scale allocations, and finer-grained
allocations will be implemented in a future patch.
libgomp/ChangeLog:
* allocator.c (MEMSPACE_ALLOC): Add PIN.
(MEMSPACE_CALLOC): Add PIN.
(MEMSPACE_REALLOC): Add PIN.
(MEMSPACE_FREE): Add PIN.
(omp_init_allocator): Don't disallow the pinned trait.
(omp_aligned_alloc): Add pinning to all MEMSPACE_* calls.
(omp_aligned_calloc): Likewise.
(omp_realloc): Likewise.
(omp_free): Likewise.
* config/linux/allocator.c: New file.
* config/nvptx/allocator.c (MEMSPACE_ALLOC): Add PIN.
(MEMSPACE_CALLOC): Add PIN.
(MEMSPACE_REALLOC): Add PIN.
(MEMSPACE_FREE): Add PIN.
* config/gcn/allocator.c (MEMSPACE_ALLOC): Add PIN.
(MEMSPACE_CALLOC): Add PIN.
(MEMSPACE_REALLOC): Add PIN.
(MEMSPACE_FREE): Add PIN.
* testsuite/libgomp.c/alloc-pinned-1.c: New test.
* testsuite/libgomp.c/alloc-pinned-2.c: New test.
* testsuite/libgomp.c/alloc-pinned-3.c: New test.
* testsuite/libgomp.c/alloc-pinned-4.c: New test.
Co-Authored-By: Thomas Schwinge <tho...@codesourcery.com>
---
libgomp/allocator.c | 66 +++++----
libgomp/config/gcn/allocator.c | 17 +--
libgomp/config/linux/allocator.c | 99 +++++++++++++
libgomp/config/nvptx/allocator.c | 17 +--
libgomp/testsuite/libgomp.c/alloc-pinned-1.c | 109 ++++++++++++++
libgomp/testsuite/libgomp.c/alloc-pinned-2.c | 114 +++++++++++++++
libgomp/testsuite/libgomp.c/alloc-pinned-3.c | 141 ++++++++++++++++++
libgomp/testsuite/libgomp.c/alloc-pinned-4.c | 143 +++++++++++++++++++
8 files changed, 665 insertions(+), 41 deletions(-)
create mode 100644 libgomp/testsuite/libgomp.c/alloc-pinned-1.c
create mode 100644 libgomp/testsuite/libgomp.c/alloc-pinned-2.c
create mode 100644 libgomp/testsuite/libgomp.c/alloc-pinned-3.c
create mode 100644 libgomp/testsuite/libgomp.c/alloc-pinned-4.c
diff --git a/libgomp/allocator.c b/libgomp/allocator.c
index a132abc45b5..6007e64f580 100644
--- a/libgomp/allocator.c
+++ b/libgomp/allocator.c
@@ -41,20 +41,21 @@
The following definitions (ab)use comma operators to avoid unused
variable errors. */
#ifndef MEMSPACE_ALLOC
-#define MEMSPACE_ALLOC(MEMSPACE, SIZE) \
- malloc (((void)(MEMSPACE), (SIZE)))
+#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
+ (PIN ? NULL : malloc (((void)(MEMSPACE), (SIZE))))
#endif
#ifndef MEMSPACE_CALLOC
-#define MEMSPACE_CALLOC(MEMSPACE, SIZE) \
- calloc (1, (((void)(MEMSPACE), (SIZE))))
+#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
+ (PIN ? NULL : calloc (1, (((void)(MEMSPACE), (SIZE)))))
#endif
#ifndef MEMSPACE_REALLOC
-#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE) \
- realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE))))
+#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
+ ((PIN) || (OLDPIN) ? NULL \
+ : realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE)))))
#endif
#ifndef MEMSPACE_FREE
-#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE) \
- free (((void)(MEMSPACE), (void)(SIZE), (ADDR)))
+#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
+ (PIN ? NULL : free (((void)(MEMSPACE), (void)(SIZE), (ADDR))))
#endif
#ifndef MEMSPACE_VALIDATE
#define MEMSPACE_VALIDATE(MEMSPACE, ACCESS) \
@@ -434,10 +435,6 @@ omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
}
#endif
- /* No support for this so far. */
- if (data.pinned)
- return omp_null_allocator;
-
ret = gomp_malloc (sizeof (struct omp_allocator_data));
*ret = data;
#ifndef HAVE_SYNC_BUILTINS
@@ -577,7 +574,8 @@ retry:
}
else
#endif
- ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size);
+ ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
+ allocator_data->pinned);
if (ptr == NULL)
{
#ifdef HAVE_SYNC_BUILTINS
@@ -614,7 +612,8 @@ retry:
memspace = (allocator_data
? allocator_data->memspace
: predefined_alloc_mapping[allocator]);
- ptr = MEMSPACE_ALLOC (memspace, new_size);
+ ptr = MEMSPACE_ALLOC (memspace, new_size,
+ allocator_data && allocator_data->pinned);
}
if (ptr == NULL)
goto fail;
@@ -646,7 +645,8 @@ fail:;
|| memkind
#endif
|| allocator_data == NULL
- || allocator_data->pool_size < ~(uintptr_t) 0)
+ || allocator_data->pool_size < ~(uintptr_t) 0
+ || allocator_data->pinned)
{
allocator = omp_default_mem_alloc;
goto retry;
@@ -697,6 +697,7 @@ omp_free (void *ptr, omp_allocator_handle_t allocator)
{
struct omp_mem_header *data;
omp_memspace_handle_t memspace = omp_default_mem_space;
+ int pinned = false;
if (ptr == NULL)
return;
@@ -738,6 +739,7 @@ omp_free (void *ptr, omp_allocator_handle_t allocator)
#endif
memspace = allocator_data->memspace;
+ pinned = allocator_data->pinned;
}
else
{
@@ -762,7 +764,7 @@ omp_free (void *ptr, omp_allocator_handle_t allocator)
memspace = predefined_alloc_mapping[data->allocator];
}
- MEMSPACE_FREE (memspace, data->ptr, data->size);
+ MEMSPACE_FREE (memspace, data->ptr, data->size, pinned);
}
ialias (omp_free)
@@ -893,7 +895,8 @@ retry:
}
else
#endif
- ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size);
+ ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size,
+ allocator_data->pinned);
if (ptr == NULL)
{
#ifdef HAVE_SYNC_BUILTINS
@@ -932,7 +935,8 @@ retry:
memspace = (allocator_data
? allocator_data->memspace
: predefined_alloc_mapping[allocator]);
- ptr = MEMSPACE_CALLOC (memspace, new_size);
+ ptr = MEMSPACE_CALLOC (memspace, new_size,
+ allocator_data && allocator_data->pinned);
}
if (ptr == NULL)
goto fail;
@@ -964,7 +968,8 @@ fail:;
|| memkind
#endif
|| allocator_data == NULL
- || allocator_data->pool_size < ~(uintptr_t) 0)
+ || allocator_data->pool_size < ~(uintptr_t) 0
+ || allocator_data->pinned)
{
allocator = omp_default_mem_alloc;
goto retry;
@@ -1176,9 +1181,13 @@ retry:
#endif
if (prev_size)
new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
- data->size, new_size);
+ data->size, new_size,
+ (free_allocator_data
+ && free_allocator_data->pinned),
+ allocator_data->pinned);
else
- new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size);
+ new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
+ allocator_data->pinned);
if (new_ptr == NULL)
{
#ifdef HAVE_SYNC_BUILTINS
@@ -1231,10 +1240,14 @@ retry:
memspace = (allocator_data
? allocator_data->memspace
: predefined_alloc_mapping[allocator]);
- new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size);
+ new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size,
+ (free_allocator_data
+ && free_allocator_data->pinned),
+ allocator_data && allocator_data->pinned);
}
if (new_ptr == NULL)
goto fail;
+
ret = (char *) new_ptr + sizeof (struct omp_mem_header);
((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
((struct omp_mem_header *) ret)[-1].size = new_size;
@@ -1264,7 +1277,8 @@ retry:
memspace = (allocator_data
? allocator_data->memspace
: predefined_alloc_mapping[allocator]);
- new_ptr = MEMSPACE_ALLOC (memspace, new_size);
+ new_ptr = MEMSPACE_ALLOC (memspace, new_size,
+ allocator_data && allocator_data->pinned);
}
if (new_ptr == NULL)
goto fail;
@@ -1319,7 +1333,8 @@ retry:
was_memspace = (free_allocator_data
? free_allocator_data->memspace
: predefined_alloc_mapping[free_allocator]);
- MEMSPACE_FREE (was_memspace, data->ptr, data->size);
+ int was_pinned = (free_allocator_data && free_allocator_data->pinned);
+ MEMSPACE_FREE (was_memspace, data->ptr, data->size, was_pinned);
}
return ret;
@@ -1337,7 +1352,8 @@ fail:;
|| memkind
#endif
|| allocator_data == NULL
- || allocator_data->pool_size < ~(uintptr_t) 0)
+ || allocator_data->pool_size < ~(uintptr_t) 0
+ || allocator_data->pinned)
{
allocator = omp_default_mem_alloc;
goto retry;
diff --git a/libgomp/config/gcn/allocator.c b/libgomp/config/gcn/allocator.c
index 151086ea225..fa701ded0ff 100644
--- a/libgomp/config/gcn/allocator.c
+++ b/libgomp/config/gcn/allocator.c
@@ -109,14 +109,15 @@ gcn_memspace_validate (omp_memspace_handle_t memspace, unsigned access)
|| access != omp_atv_all);
}
-#define MEMSPACE_ALLOC(MEMSPACE, SIZE) \
- gcn_memspace_alloc (MEMSPACE, SIZE)
-#define MEMSPACE_CALLOC(MEMSPACE, SIZE) \
- gcn_memspace_calloc (MEMSPACE, SIZE)
-#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE) \
- gcn_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, SIZE)
-#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE) \
- gcn_memspace_free (MEMSPACE, ADDR, SIZE)
+#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
+ gcn_memspace_alloc (MEMSPACE, ((void)(PIN), (SIZE)))
+#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
+ gcn_memspace_calloc (MEMSPACE, ((void)(PIN), (SIZE)))
+#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
+ gcn_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, \
+ ((void)(PIN), (void)(OLDPIN), (SIZE)))
+#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
+ gcn_memspace_free (MEMSPACE, ADDR, ((void)(PIN), (SIZE)))
#define MEMSPACE_VALIDATE(MEMSPACE, ACCESS) \
gcn_memspace_validate (MEMSPACE, ACCESS)
diff --git a/libgomp/config/linux/allocator.c b/libgomp/config/linux/allocator.c
index 64b1b4b9623..edcde9f1e81 100644
--- a/libgomp/config/linux/allocator.c
+++ b/libgomp/config/linux/allocator.c
@@ -34,4 +34,103 @@
#define LIBGOMP_USE_LIBNUMA
#endif
+/* Implement malloc routines that can handle pinned memory on Linux.
+
+ It's possible to use mlock on any heap memory, but using munlock is
+ problematic if there are multiple pinned allocations on the same page.
+ Tracking all that manually would be possible, but adds overhead. This may
+ be worth it if there are a lot of small allocations getting pinned, but
+ this seems less likely in a HPC application.
+
+ Instead we optimize for large pinned allocations, and use mmap to ensure
+ that two pinned allocations don't share the same page. This also means
+ that large allocations don't pin extra pages by being poorly aligned. */
+
+#define _GNU_SOURCE
+#include <sys/mman.h>
+#include <string.h>
+#include "libgomp.h"
+
+static void *
+linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin)
+{
+ (void)memspace;
+
+ if (pin)
+ {
+ void *addr = mmap (NULL, size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (addr == MAP_FAILED)
+ return NULL;
+
+ if (mlock (addr, size))
+ {
+ gomp_debug (0, "libgomp: failed to pin memory (ulimit too low?)\n");
+ munmap (addr, size);
+ return NULL;
+ }
+
+ return addr;
+ }
+ else
+ return malloc (size);
+}
+
+static void *
+linux_memspace_calloc (omp_memspace_handle_t memspace, size_t size, int pin)
+{
+ if (pin)
+ return linux_memspace_alloc (memspace, size, pin);
+ else
+ return calloc (1, size);
+}
+
+static void
+linux_memspace_free (omp_memspace_handle_t memspace, void *addr, size_t size,
+ int pin)
+{
+ (void)memspace;
+
+ if (pin)
+ munmap (addr, size);
+ else
+ free (addr);
+}
+
+static void *
+linux_memspace_realloc (omp_memspace_handle_t memspace, void *addr,
+ size_t oldsize, size_t size, int oldpin, int pin)
+{
+ if (oldpin && pin)
+ {
+ void *newaddr = mremap (addr, oldsize, size, MREMAP_MAYMOVE);
+ if (newaddr == MAP_FAILED)
+ return NULL;
+
+ return newaddr;
+ }
+ else if (oldpin || pin)
+ {
+ void *newaddr = linux_memspace_alloc (memspace, size, pin);
+ if (newaddr)
+ {
+ memcpy (newaddr, addr, oldsize < size ? oldsize : size);
+ linux_memspace_free (memspace, addr, oldsize, oldpin);
+ }
+
+ return newaddr;
+ }
+ else
+ return realloc (addr, size);
+}
+
+#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
+ linux_memspace_alloc (MEMSPACE, SIZE, PIN)
+#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
+ linux_memspace_calloc (MEMSPACE, SIZE, PIN)
+#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
+ linux_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN)
+#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
+ linux_memspace_free (MEMSPACE, ADDR, SIZE, PIN)
+
#include "../../allocator.c"
diff --git a/libgomp/config/nvptx/allocator.c b/libgomp/config/nvptx/allocator.c
index f19ac28d32a..607ef0a77b0 100644
--- a/libgomp/config/nvptx/allocator.c
+++ b/libgomp/config/nvptx/allocator.c
@@ -117,14 +117,15 @@ nvptx_memspace_validate (omp_memspace_handle_t memspace, unsigned access)
|| access != omp_atv_all);
}
-#define MEMSPACE_ALLOC(MEMSPACE, SIZE) \
- nvptx_memspace_alloc (MEMSPACE, SIZE)
-#define MEMSPACE_CALLOC(MEMSPACE, SIZE) \
- nvptx_memspace_calloc (MEMSPACE, SIZE)
-#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE) \
- nvptx_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, SIZE)
-#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE) \
- nvptx_memspace_free (MEMSPACE, ADDR, SIZE)
+#define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
+ nvptx_memspace_alloc (MEMSPACE, ((void)(PIN), (SIZE)))
+#define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
+ nvptx_memspace_calloc (MEMSPACE, ((void)(PIN), (SIZE)))
+#define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
+ nvptx_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, \
+ ((void)(OLDPIN), (void)(PIN), (SIZE)))
+#define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
+ nvptx_memspace_free (MEMSPACE, ADDR, ((void)(PIN), (SIZE)))
#define MEMSPACE_VALIDATE(MEMSPACE, ACCESS) \
nvptx_memspace_validate (MEMSPACE, ACCESS)
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-1.c b/libgomp/testsuite/libgomp.c/alloc-pinned-1.c
new file mode 100644
index 00000000000..8731090bb54
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-1.c
@@ -0,0 +1,109 @@
+/* { dg-do run } */
+
+/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */
+
+/* Test that pinned memory works. */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+#include <sys/resource.h>
+
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+#define CHECK_SIZE(SIZE) { \
+ struct rlimit limit; \
+ if (getrlimit (RLIMIT_MEMLOCK, &limit) \
+ || limit.rlim_cur <= SIZE) \
+ fprintf (stderr, "unsufficient lockable memory; please increase ulimit\n"); \
+ }
+
+int
+get_pinned_mem ()
+{
+ int pid = getpid ();
+ char buf[100];
+ sprintf (buf, "/proc/%d/status", pid);
+
+ FILE *proc = fopen (buf, "r");
+ if (!proc)
+ abort ();
+ while (fgets (buf, 100, proc))
+ {
+ int val;
+ if (sscanf (buf, "VmLck: %d", &val))
+ {
+ fclose (proc);
+ return val;
+ }
+ }
+ abort ();
+}
+#else
+#define PAGE_SIZE 1 /* unknown */
+#define CHECK_SIZE(SIZE) fprintf (stderr, "OS unsupported\n");
+
+int
+get_pinned_mem ()
+{
+ return 0;
+}
+#endif
+
+static void
+verify0 (char *p, size_t s)
+{
+ for (size_t i = 0; i < s; ++i)
+ if (p[i] != 0)
+ abort ();
+}
+
+#include <omp.h>
+
+int
+main ()
+{
+ /* Allocate at least a page each time, allowing space for overhead,
+ but stay within the ulimit. */
+ const int SIZE = PAGE_SIZE - 128;
+ CHECK_SIZE (SIZE * 5);
+
+ const omp_alloctrait_t traits[] = {
+ { omp_atk_pinned, 1 }
+ };
+ omp_allocator_handle_t allocator = omp_init_allocator (omp_default_mem_space,
+ 1, traits);
+
+ // Sanity check
+ if (get_pinned_mem () != 0)
+ abort ();
+
+ void *p = omp_alloc (SIZE, allocator);
+ if (!p)
+ abort ();
+
+ int amount = get_pinned_mem ();
+ if (amount == 0)
+ abort ();
+
+ p = omp_realloc (p, SIZE * 2, allocator, allocator);
+
+ int amount2 = get_pinned_mem ();
+ if (amount2 <= amount)
+ abort ();
+
+ /* SIZE*2 ensures that it doesn't slot into the space possibly
+ vacated by realloc. */
+ p = omp_calloc (1, SIZE * 2, allocator);
+
+ if (get_pinned_mem () <= amount2)
+ abort ();
+
+ verify0 (p, SIZE * 2);
+
+ return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-2.c b/libgomp/testsuite/libgomp.c/alloc-pinned-2.c
new file mode 100644
index 00000000000..27d2403960d
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-2.c
@@ -0,0 +1,114 @@
+/* { dg-do run } */
+
+/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */
+
+/* Test that pinned memory works (pool_size code path). */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+#include <sys/resource.h>
+
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+#define CHECK_SIZE(SIZE) { \
+ struct rlimit limit; \
+ if (getrlimit (RLIMIT_MEMLOCK, &limit) \
+ || limit.rlim_cur <= SIZE) \
+ fprintf (stderr, "unsufficient lockable memory; please increase ulimit\n"); \
+ }
+
+int
+get_pinned_mem ()
+{
+ int pid = getpid ();
+ char buf[100];
+ sprintf (buf, "/proc/%d/status", pid);
+
+ FILE *proc = fopen (buf, "r");
+ if (!proc)
+ abort ();
+ while (fgets (buf, 100, proc))
+ {
+ int val;
+ if (sscanf (buf, "VmLck: %d", &val))
+ {
+ fclose (proc);
+ return val;
+ }
+ }
+ abort ();
+}
+#else
+#define PAGE_SIZE 1 /* unknown */
+#define CHECK_SIZE(SIZE) fprintf (stderr, "OS unsupported\n");
+
+int
+get_pinned_mem ()
+{
+ return 0;
+}
+#endif
+
+static void
+verify0 (char *p, size_t s)
+{
+ for (size_t i = 0; i < s; ++i)
+ if (p[i] != 0)
+ abort ();
+}
+
+#include <omp.h>
+
+int
+main ()
+{
+ /* Allocate at least a page each time, allowing space for overhead,
+ but stay within the ulimit. */
+ const int SIZE = PAGE_SIZE - 128;
+ CHECK_SIZE (SIZE * 5);
+
+ const omp_alloctrait_t traits[] = {
+ { omp_atk_pinned, 1 },
+ { omp_atk_pool_size, SIZE * 8 }
+ };
+ omp_allocator_handle_t allocator = omp_init_allocator (omp_default_mem_space,
+ 2, traits);
+
+ // Sanity check
+ if (get_pinned_mem () != 0)
+ abort ();
+
+ void *p = omp_alloc (SIZE, allocator);
+ if (!p)
+ abort ();
+
+ int amount = get_pinned_mem ();
+ if (amount == 0)
+ abort ();
+
+ p = omp_realloc (p, SIZE * 2, allocator, allocator);
+ if (!p)
+ abort ();
+
+ int amount2 = get_pinned_mem ();
+ if (amount2 <= amount)
+ abort ();
+
+ /* SIZE*2 ensures that it doesn't slot into the space possibly
+ vacated by realloc. */
+ p = omp_calloc (1, SIZE * 2, allocator);
+ if (!p)
+ abort ();
+
+ if (get_pinned_mem () <= amount2)
+ abort ();
+
+ verify0 (p, SIZE * 2);
+
+ return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-3.c b/libgomp/testsuite/libgomp.c/alloc-pinned-3.c
new file mode 100644
index 00000000000..6521f135c46
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-3.c
@@ -0,0 +1,141 @@
+/* { dg-do run } */
+
+/* Test that pinned memory fails correctly. */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+#include <sys/resource.h>
+
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+
+int
+get_pinned_mem ()
+{
+ int pid = getpid ();
+ char buf[100];
+ sprintf (buf, "/proc/%d/status", pid);
+
+ FILE *proc = fopen (buf, "r");
+ if (!proc)
+ abort ();
+ while (fgets (buf, 100, proc))
+ {
+ int val;
+ if (sscanf (buf, "VmLck: %d", &val))
+ {
+ fclose (proc);
+ return val;
+ }
+ }
+ abort ();
+}
+
+void
+set_pin_limit (int size)
+{
+ struct rlimit limit;
+ if (getrlimit (RLIMIT_MEMLOCK, &limit))
+ abort ();
+ limit.rlim_cur = (limit.rlim_max < size ? limit.rlim_max : size);
+ if (setrlimit (RLIMIT_MEMLOCK, &limit))
+ abort ();
+}
+#else
+int
+#define PAGE_SIZE 10000 * 1024 /* unknown */
+
+get_pinned_mem ()
+{
+ return 0;
+}
+
+void
+set_pin_limit ()
+{
+}
+#endif
+
+static void
+verify0 (char *p, size_t s)
+{
+ for (size_t i = 0; i < s; ++i)
+ if (p[i] != 0)
+ abort ();
+}
+
+#include <omp.h>
+
+int
+main ()
+{
+ /* This needs to be large enough to cover multiple pages. */
+ const int SIZE = PAGE_SIZE * 4;
+
+ /* Pinned memory, no fallback. */
+ const omp_alloctrait_t traits1[] = {
+ { omp_atk_pinned, 1 },
+ { omp_atk_fallback, omp_atv_null_fb }
+ };
+ omp_allocator_handle_t allocator1 = omp_init_allocator (omp_default_mem_space,
+ 2, traits1);
+
+ /* Pinned memory, plain memory fallback. */
+ const omp_alloctrait_t traits2[] = {
+ { omp_atk_pinned, 1 },
+ { omp_atk_fallback, omp_atv_default_mem_fb }
+ };
+ omp_allocator_handle_t allocator2 = omp_init_allocator (omp_default_mem_space,
+ 2, traits2);
+
+ /* Ensure that the limit is smaller than the allocation. */
+ set_pin_limit (SIZE / 2);
+
+ // Sanity check
+ if (get_pinned_mem () != 0)
+ abort ();
+
+ // Should fail
+ void *p = omp_alloc (SIZE, allocator1);
+ if (p)
+ abort ();
+
+ // Should fail
+ p = omp_calloc (1, SIZE, allocator1);
+ if (p)
+ abort ();
+
+ // Should fall back
+ p = omp_alloc (SIZE, allocator2);
+ if (!p)
+ abort ();
+
+ // Should fall back
+ p = omp_calloc (1, SIZE, allocator2);
+ if (!p)
+ abort ();
+ verify0 (p, SIZE);
+
+ // Should fail to realloc
+ void *notpinned = omp_alloc (SIZE, omp_default_mem_alloc);
+ p = omp_realloc (notpinned, SIZE, allocator1, omp_default_mem_alloc);
+ if (!notpinned || p)
+ abort ();
+
+ // Should fall back to no realloc needed
+ p = omp_realloc (notpinned, SIZE, allocator2, omp_default_mem_alloc);
+ if (p != notpinned)
+ abort ();
+
+ // No memory should have been pinned
+ int amount = get_pinned_mem ();
+ if (amount != 0)
+ abort ();
+
+ return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c/alloc-pinned-4.c b/libgomp/testsuite/libgomp.c/alloc-pinned-4.c
new file mode 100644
index 00000000000..51917557c91
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/alloc-pinned-4.c
@@ -0,0 +1,143 @@
+/* { dg-do run } */
+
+/* Test that pinned memory fails correctly, pool_size code path. */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+#include <sys/resource.h>
+
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+
+int
+get_pinned_mem ()
+{
+ int pid = getpid ();
+ char buf[100];
+ sprintf (buf, "/proc/%d/status", pid);
+
+ FILE *proc = fopen (buf, "r");
+ if (!proc)
+ abort ();
+ while (fgets (buf, 100, proc))
+ {
+ int val;
+ if (sscanf (buf, "VmLck: %d", &val))
+ {
+ fclose (proc);
+ return val;
+ }
+ }
+ abort ();
+}
+
+void
+set_pin_limit (int size)
+{
+ struct rlimit limit;
+ if (getrlimit (RLIMIT_MEMLOCK, &limit))
+ abort ();
+ limit.rlim_cur = (limit.rlim_max < size ? limit.rlim_max : size);
+ if (setrlimit (RLIMIT_MEMLOCK, &limit))
+ abort ();
+}
+#else
+int
+#define PAGE_SIZE 10000 * 1024 /* unknown */
+
+get_pinned_mem ()
+{
+ return 0;
+}
+
+void
+set_pin_limit ()
+{
+}
+#endif
+
+static void
+verify0 (char *p, size_t s)
+{
+ for (size_t i = 0; i < s; ++i)
+ if (p[i] != 0)
+ abort ();
+}
+
+#include <omp.h>
+
+int
+main ()
+{
+ /* This needs to be large enough to cover multiple pages. */
+ const int SIZE = PAGE_SIZE * 4;
+
+ /* Pinned memory, no fallback. */
+ const omp_alloctrait_t traits1[] = {
+ { omp_atk_pinned, 1 },
+ { omp_atk_fallback, omp_atv_null_fb },
+ { omp_atk_pool_size, SIZE * 8 }
+ };
+ omp_allocator_handle_t allocator1 = omp_init_allocator (omp_default_mem_space,
+ 3, traits1);
+
+ /* Pinned memory, plain memory fallback. */
+ const omp_alloctrait_t traits2[] = {
+ { omp_atk_pinned, 1 },
+ { omp_atk_fallback, omp_atv_default_mem_fb },
+ { omp_atk_pool_size, SIZE * 8 }
+ };
+ omp_allocator_handle_t allocator2 = omp_init_allocator (omp_default_mem_space,
+ 3, traits2);
+
+ /* Ensure that the limit is smaller than the allocation. */
+ set_pin_limit (SIZE / 2);
+
+ // Sanity check
+ if (get_pinned_mem () != 0)
+ abort ();
+
+ // Should fail
+ void *p = omp_alloc (SIZE, allocator1);
+ if (p)
+ abort ();
+
+ // Should fail
+ p = omp_calloc (1, SIZE, allocator1);
+ if (p)
+ abort ();
+
+ // Should fall back
+ p = omp_alloc (SIZE, allocator2);
+ if (!p)
+ abort ();
+
+ // Should fall back
+ p = omp_calloc (1, SIZE, allocator2);
+ if (!p)
+ abort ();
+ verify0 (p, SIZE);
+
+ // Should fail to realloc
+ void *notpinned = omp_alloc (SIZE, omp_default_mem_alloc);
+ p = omp_realloc (notpinned, SIZE, allocator1, omp_default_mem_alloc);
+ if (!notpinned || p)
+ abort ();
+
+ // Should fall back to no realloc needed
+ p = omp_realloc (notpinned, SIZE, allocator2, omp_default_mem_alloc);
+ if (p != notpinned)
+ abort ();
+
+ // No memory should have been pinned
+ int amount = get_pinned_mem ();
+ if (amount != 0)
+ abort ();
+
+ return 0;
+}
