Move gather_after_split_folio_orders() to vm_util.c as a helper function in preparation for implementing checks for mTHP collapse. While we are at it, rename this function to indicate that it is not only used for large folio splits.
No functional changes. Signed-off-by: Baolin Wang <[email protected]> --- .../selftests/mm/split_huge_page_test.c | 125 +----------------- tools/testing/selftests/mm/vm_util.c | 119 +++++++++++++++++ tools/testing/selftests/mm/vm_util.h | 2 + 3 files changed, 122 insertions(+), 124 deletions(-) diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 4cc70873a674..86a603692826 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -104,129 +104,6 @@ static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, return false; } -static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd, - uint64_t *flags) -{ - unsigned long pfn; - - pfn = pagemap_get_pfn(pagemap_fd, vaddr); - - /* non-present PFN */ - if (pfn == -1UL) - return 1; - - if (pageflags_get(pfn, kpageflags_fd, flags)) - return -1; - - return 0; -} - -/* - * gather_after_split_folio_orders - scan through [vaddr_start, len) and record - * folio orders - * - * @vaddr_start: start vaddr - * @len: range length - * @pagemap_fd: file descriptor to /proc/<pid>/pagemap - * @kpageflags_fd: file descriptor to /proc/kpageflags - * @orders: output folio order array - * @nr_orders: folio order array size - * - * gather_after_split_folio_orders() scan through [vaddr_start, len) and check - * all folios within the range and record their orders. All order-0 pages will - * be recorded. Non-present vaddr is skipped. - * - * NOTE: the function is used to check folio orders after a split is performed, - * so it assumes [vaddr_start, len) fully maps to after-split folios within that - * range. - * - * Return: 0 - no error, -1 - unhandled cases - */ -static int gather_after_split_folio_orders(char *vaddr_start, size_t len, - int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) -{ - uint64_t page_flags = 0; - int cur_order = -1; - char *vaddr; - - if (pagemap_fd == -1 || kpageflags_fd == -1) - return -1; - if (!orders) - return -1; - if (nr_orders <= 0) - return -1; - - for (vaddr = vaddr_start; vaddr < vaddr_start + len;) { - char *next_folio_vaddr; - int status; - - status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd, - &page_flags); - if (status < 0) - return -1; - - /* skip non present vaddr */ - if (status == 1) { - vaddr += psize(); - continue; - } - - /* all order-0 pages with possible false postive (non folio) */ - if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { - orders[0]++; - vaddr += psize(); - continue; - } - - /* skip non thp compound pages */ - if (!(page_flags & KPF_THP)) { - vaddr += psize(); - continue; - } - - /* vpn points to part of a THP at this point */ - if (page_flags & KPF_COMPOUND_HEAD) - cur_order = 1; - else { - vaddr += psize(); - continue; - } - - next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); - - if (next_folio_vaddr >= vaddr_start + len) - break; - - while ((status = vaddr_pageflags_get(next_folio_vaddr, - pagemap_fd, kpageflags_fd, - &page_flags)) >= 0) { - /* - * non present vaddr, next compound head page, or - * order-0 page - */ - if (status == 1 || - (page_flags & KPF_COMPOUND_HEAD) || - !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { - if (cur_order < nr_orders) { - orders[cur_order]++; - cur_order = -1; - vaddr = next_folio_vaddr; - } - break; - } - - cur_order++; - next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); - } - - if (status < 0) - return status; - } - if (cur_order > 0 && cur_order < nr_orders) - orders[cur_order]++; - return 0; -} - static int check_after_split_folio_orders(char *vaddr_start, size_t len, int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) { @@ -240,7 +117,7 @@ static int check_after_split_folio_orders(char *vaddr_start, size_t len, ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders"); memset(vaddr_orders, 0, sizeof(int) * nr_orders); - status = gather_after_split_folio_orders(vaddr_start, len, pagemap_fd, + status = gather_folio_orders(vaddr_start, len, pagemap_fd, kpageflags_fd, vaddr_orders, nr_orders); if (status) ksft_exit_fail_msg("gather folio info failed\n"); diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index d92c819f3717..09e5d5cabe21 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -194,6 +194,125 @@ unsigned long rss_anon(void) return rss_anon; } +static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd, + uint64_t *flags) +{ + unsigned long pfn; + + pfn = pagemap_get_pfn(pagemap_fd, vaddr); + + /* non-present PFN */ + if (pfn == -1UL) + return 1; + + if (pageflags_get(pfn, kpageflags_fd, flags)) + return -1; + + return 0; +} + +/* + * gather_folio_orders - scan through [vaddr_start, len) and record + * folio orders + * + * @vaddr_start: start vaddr + * @len: range length + * @pagemap_fd: file descriptor to /proc/<pid>/pagemap + * @kpageflags_fd: file descriptor to /proc/kpageflags + * @orders: output folio order array + * @nr_orders: folio order array size + * + * gather_folio_orders() scan through [vaddr_start, len) and check + * all folios within the range and record their orders. All order-0 pages will + * be recorded. Non-present vaddr is skipped. + * + * Return: 0 - no error, -1 - unhandled cases + */ +int gather_folio_orders(char *vaddr_start, size_t len, + int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) +{ + uint64_t page_flags = 0; + int cur_order = -1; + char *vaddr; + + if (pagemap_fd == -1 || kpageflags_fd == -1) + return -1; + if (!orders) + return -1; + if (nr_orders <= 0) + return -1; + + for (vaddr = vaddr_start; vaddr < vaddr_start + len;) { + char *next_folio_vaddr; + int status; + + status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd, + &page_flags); + if (status < 0) + return -1; + + /* skip non present vaddr */ + if (status == 1) { + vaddr += psize(); + continue; + } + + /* all order-0 pages with possible false postive (non folio) */ + if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { + orders[0]++; + vaddr += psize(); + continue; + } + + /* skip non thp compound pages */ + if (!(page_flags & KPF_THP)) { + vaddr += psize(); + continue; + } + + /* vpn points to part of a THP at this point */ + if (page_flags & KPF_COMPOUND_HEAD) + cur_order = 1; + else { + vaddr += psize(); + continue; + } + + next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); + + if (next_folio_vaddr >= vaddr_start + len) + break; + + while ((status = vaddr_pageflags_get(next_folio_vaddr, + pagemap_fd, kpageflags_fd, + &page_flags)) >= 0) { + /* + * non present vaddr, next compound head page, or + * order-0 page + */ + if (status == 1 || + (page_flags & KPF_COMPOUND_HEAD) || + !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { + if (cur_order < nr_orders) { + orders[cur_order]++; + cur_order = -1; + vaddr = next_folio_vaddr; + } + break; + } + + cur_order++; + next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); + } + + if (status < 0) + return status; + } + if (cur_order > 0 && cur_order < nr_orders) + orders[cur_order]++; + return 0; +} + char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) { int ret; diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index c7ae28dfda63..20af3e574cd8 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -95,6 +95,8 @@ bool check_huge_file(void *addr, unsigned long size, int nr_hpages, uint64_t hpa bool check_huge_shmem(void *addr, unsigned long size, int nr_hpages, uint64_t hpage_size); int64_t allocate_transhuge(void *ptr, int pagemap_fd); int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags); +int gather_folio_orders(char *vaddr_start, size_t len, + int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders); int uffd_register(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor); -- 2.47.3
