Add a new GUP selftest which uses kselftest_harness.h. Cover 12 mapping configurations: THP enabled, THP disabled and HugeTLB, each across private/shared mappings and with/without FOLL_WRITE. Run 5 test cases for every variant: get_user_pages, get_user_pages_fast, pin_user_pages, pin_user_pages_fast and pin_user_pages_longterm.
Choose the number of default hugeTLB pages using a 256 MiB target, with a minimum of 1 page and derive the mapping size from that number. This avoids reserving excess memory when the hugeTLB page size is too large and retains 128 pages for the most common case of 2MiB hugeTLB pages. Sweep four nr_pages_per_call values for each test: 1, 512, 123 and all pages. This preserves the coverage previously provided by run_gup_matrix(): 12 mapping combinations x 5 GUP/PUP operations x 4 batch sizes. In total the selftest reports 60 TAP cases and issues 240 ioctls. Do not carry DUMP_USER_PAGES_TEST into the new selftest because its output is written to the kernel log and the selftest does not verify that output. Add the new gup binary to the selftests/mm build, run_vmtests.sh and MAINTAINERS. Update mm/Kconfig to describe the benchmark and selftest split. Suggested-by: David Hildenbrand (Arm) <[email protected]> Acked-by: Mike Rapoport (Microsoft) <[email protected]> Tested-by: Muhammad Usama Anjum <[email protected]> Signed-off-by: Sarthak Sharma <[email protected]> --- MAINTAINERS | 1 + mm/Kconfig | 21 +- tools/testing/selftests/mm/Makefile | 1 + tools/testing/selftests/mm/gup.c | 287 ++++++++++++++++++++++ tools/testing/selftests/mm/run_vmtests.sh | 1 + 5 files changed, 300 insertions(+), 11 deletions(-) create mode 100644 tools/testing/selftests/mm/gup.c diff --git a/MAINTAINERS b/MAINTAINERS index d7a146b093de..1ca7de3e440e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17189,6 +17189,7 @@ F: mm/gup.c F: mm/gup_test.c F: mm/gup_test.h F: tools/mm/gup_bench.c +F: tools/testing/selftests/mm/gup.c F: tools/testing/selftests/mm/gup_longterm.c MEMORY MANAGEMENT - KSM (Kernel Samepage Merging) diff --git a/mm/Kconfig b/mm/Kconfig index c1ddf59c0d71..79163b7d795a 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1287,24 +1287,23 @@ config PERCPU_STATS be used to help understand percpu memory usage. config GUP_TEST - bool "Enable infrastructure for get_user_pages()-related unit tests" + bool "Enable infrastructure for get_user_pages()-related unit tests and benchmarks" depends on DEBUG_FS help Provides /sys/kernel/debug/gup_test, which in turn provides a way - to make ioctl calls that can launch kernel-based unit tests for - the get_user_pages*() and pin_user_pages*() family of API calls. + to make ioctl calls that can launch kernel-based unit tests and + benchmarks for the get_user_pages*() and pin_user_pages*() families + of API calls. - These tests include benchmark testing of the _fast variants of - get_user_pages*() and pin_user_pages*(), as well as smoke tests of + These include benchmark testing of the _fast variants of + get_user_pages*() and pin_user_pages*(), as well as tests of the non-_fast variants. - There is also a sub-test that allows running dump_page() on any - of up to eight pages (selected by command line args) within the - range of user-space addresses. These pages are either pinned via - pin_user_pages*(), or pinned via get_user_pages*(), as specified - by other command line arguments. + There is also a test that allows running dump_page() on any of up + to eight pages within the range of user-space addresses. These + pages are either acquired via pin_user_pages*() or get_user_pages*(). - See tools/testing/selftests/mm/gup_test.c + See tools/testing/selftests/mm/gup.c and tools/mm/gup_bench.c. comment "GUP_TEST needs to have DEBUG_FS enabled" depends on !GUP_TEST && !DEBUG_FS diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 11ca9b11fef1..9c03624fd293 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -58,6 +58,7 @@ endif TEST_GEN_FILES = cow TEST_GEN_FILES += compaction_test +TEST_GEN_FILES += gup TEST_GEN_FILES += gup_longterm TEST_GEN_FILES += hmm-tests TEST_GEN_FILES += hugetlb-madvise diff --git a/tools/testing/selftests/mm/gup.c b/tools/testing/selftests/mm/gup.c new file mode 100644 index 000000000000..6a0dd415fb9a --- /dev/null +++ b/tools/testing/selftests/mm/gup.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0 +#define __SANE_USERSPACE_TYPES__ // Use ll64 +#include <fcntl.h> +#include <errno.h> +#include <stdbool.h> +#include <string.h> +#include <unistd.h> +#include <dirent.h> +#include <sys/ioctl.h> +#include <sys/mman.h> +#include <mm/gup_test.h> +#include "vm_util.h" +#include "kselftest_harness.h" + +#define MB (1UL << 20) + +/* Just the flags we need, copied from the kernel internals. */ +#define FOLL_WRITE 0x01 /* check pte is writable */ + +/* Page counts exercising single, THP-batch, partial, and full-mapping GUP. */ +static const int nr_pages_list[] = { 1, 512, 123, -1 }; + +#define GUP_TEST_FILE "/sys/kernel/debug/gup_test" +#define HUGETLB_TARGET_SIZE (256 * MB) + +static unsigned long nr_huge_pages, hp_size; +static bool hugetlb_setup_succeeded; + +FIXTURE(gup_test) +{ + int gup_fd; + char *addr; + unsigned long size; +}; + +FIXTURE_VARIANT(gup_test) +{ + bool thp; + bool hugetlb; + bool write; + bool shared; +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write) +{ + .thp = false, + .hugetlb = false, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read) +{ + .thp = false, + .hugetlb = false, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write_thp) +{ + .thp = true, + .hugetlb = false, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read_thp) +{ + .thp = true, + .hugetlb = false, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_write_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = true, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, private_read_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = false, + .shared = false, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write) +{ + .thp = false, + .hugetlb = false, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read) +{ + .thp = false, + .hugetlb = false, + .write = false, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write_thp) +{ + .thp = true, + .hugetlb = false, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read_thp) +{ + .thp = true, + .hugetlb = false, + .write = false, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_write_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = true, + .shared = true, +}; + +FIXTURE_VARIANT_ADD(gup_test, shared_read_hugetlb) +{ + .thp = false, + .hugetlb = true, + .write = false, + .shared = true, +}; + +FIXTURE_SETUP(gup_test) +{ + int mmap_flags = MAP_PRIVATE; + int zero_fd; + char *p; + + self->size = 128 * MB; + + if (variant->hugetlb) { + if (!hp_size) + SKIP(return, "HugeTLB not available\n"); + + if (!hugetlb_setup_succeeded || + hugetlb_free_default_pages() < nr_huge_pages) + SKIP(return, "Not enough huge pages\n"); + + self->size = nr_huge_pages * hp_size; + mmap_flags |= (MAP_HUGETLB | MAP_ANONYMOUS); + } + + if (variant->shared) + mmap_flags = (mmap_flags & ~MAP_PRIVATE) | MAP_SHARED; + + /* zero_fd has to be >= 0. Already checked in main() */ + zero_fd = open("/dev/zero", O_RDWR); + ASSERT_GE(zero_fd, 0); + + /* gup_fd has to be >= 0. Already checked in main() */ + self->gup_fd = open(GUP_TEST_FILE, O_RDWR); + ASSERT_GE(self->gup_fd, 0); + + self->addr = mmap(NULL, self->size, PROT_READ | PROT_WRITE, + mmap_flags, zero_fd, 0); + + ASSERT_NE(self->addr, MAP_FAILED) { + int err = errno; + + close(zero_fd); + close(self->gup_fd); + TH_LOG("mmap failed: %s", strerror(err)); + } + close(zero_fd); + + if (variant->thp) + madvise(self->addr, self->size, MADV_HUGEPAGE); + else if (!variant->hugetlb) + madvise(self->addr, self->size, MADV_NOHUGEPAGE); + + for (p = self->addr; (unsigned long)p < (unsigned long)self->addr + + self->size; p += psize()) + p[0] = 0; +} + +FIXTURE_TEARDOWN(gup_test) +{ + munmap(self->addr, self->size); + close(self->gup_fd); +} + +static void run_gup_cmd(struct __test_metadata *_metadata, + FIXTURE_DATA(gup_test) *self, + const FIXTURE_VARIANT(gup_test) *variant, + unsigned long command) +{ + int i; + + for (i = 0; i < (int)ARRAY_SIZE(nr_pages_list); i++) { + struct gup_test gup = { + .addr = (unsigned long)self->addr, + .size = self->size, + .nr_pages_per_call = nr_pages_list[i] < 0 ? + self->size / psize() : nr_pages_list[i], + .gup_flags = variant->write ? FOLL_WRITE : 0, + }; + + TH_LOG("nr_pages_per_call=%u", gup.nr_pages_per_call); + ASSERT_EQ(ioctl(self->gup_fd, command, &gup), 0); + ASSERT_EQ(gup.size, self->size); + } +} + +TEST_F(gup_test, get_user_pages) +{ + run_gup_cmd(_metadata, self, variant, GUP_BASIC_TEST); +} + +TEST_F(gup_test, pin_user_pages) +{ + run_gup_cmd(_metadata, self, variant, PIN_BASIC_TEST); +} + +TEST_F(gup_test, get_user_pages_fast) +{ + run_gup_cmd(_metadata, self, variant, GUP_FAST_BENCHMARK); +} + +TEST_F(gup_test, pin_user_pages_fast) +{ + run_gup_cmd(_metadata, self, variant, PIN_FAST_BENCHMARK); +} + +TEST_F(gup_test, pin_user_pages_longterm) +{ + run_gup_cmd(_metadata, self, variant, PIN_LONGTERM_BENCHMARK); +} + +int main(int argc, char **argv) +{ + char *file = "/dev/zero"; + int fd; + + fd = open(file, O_RDWR); + if (fd < 0) { + ksft_print_header(); + ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno)); + } + close(fd); + + fd = open(GUP_TEST_FILE, O_RDWR); + if (fd == -1) { + ksft_print_header(); + if (errno == EACCES) + ksft_exit_skip("Please run this test as root\n"); + if (errno == ENOENT) { + DIR *debugfs = opendir("/sys/kernel/debug"); + + if (!debugfs) { + ksft_exit_skip("Mount debugfs at /sys/kernel/debug\n"); + } else { + closedir(debugfs); + ksft_exit_skip("Check CONFIG_GUP_TEST in kernel config\n"); + } + } + ksft_exit_fail_msg("Failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno)); + } + close(fd); + + hp_size = default_huge_page_size(); + if (hp_size) { + nr_huge_pages = HUGETLB_TARGET_SIZE / hp_size; + if (!nr_huge_pages) + nr_huge_pages = 1; + + hugetlb_setup_succeeded = hugetlb_setup_default(nr_huge_pages); + } + + return test_harness_run(argc, argv); +} diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 8f1e828e4f39..ae0ab5efabae 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -251,6 +251,7 @@ fi CATEGORY="mmap" run_test ./map_fixed_noreplace +CATEGORY="gup_test" run_test ./gup CATEGORY="gup_test" run_test ./gup_longterm CATEGORY="userfaultfd" run_test ./uffd-unit-tests -- 2.53.0
