From: Jonathan Lemon <jonathan.le...@gmail.com>
1) Rename functions to reflect what they are actually doing.
2) Unify the condition to keep a page.
3) When page can't be kept in cache, fallback to releasing page to page
allocator in one place, instead of calling it from multiple conditions,
and reuse __page_pool_return_page().
Signed-off-by: Jonathan Lemon <jonathan.le...@gmail.com>
Signed-off-by: Saeed Mahameed <sae...@mellanox.com>
---
net/core/page_pool.c | 38 +++++++++++++++++++-------------------
1 file changed, 19 insertions(+), 19 deletions(-)
diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 8120aec999ce..65680aaa0818 100644
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -258,6 +258,7 @@ static bool __page_pool_recycle_into_ring(struct page_pool
*pool,
struct page *page)
{
int ret;
+
/* BH protection not needed if current is serving softirq */
if (in_serving_softirq())
ret = ptr_ring_produce(&pool->ring, page);
@@ -272,8 +273,8 @@ static bool __page_pool_recycle_into_ring(struct page_pool
*pool,
*
* Caller must provide appropriate safe context.
*/
-static bool __page_pool_recycle_direct(struct page *page,
- struct page_pool *pool)
+static bool __page_pool_recycle_into_cache(struct page *page,
+ struct page_pool *pool)
{
if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE))
return false;
@@ -283,15 +284,18 @@ static bool __page_pool_recycle_direct(struct page *page,
return true;
}
-/* page is NOT reusable when:
- * 1) allocated when system is under some pressure. (page_is_pfmemalloc)
- * 2) belongs to a different NUMA node than pool->p.nid.
+/* Keep page in caches only if page:
+ * 1) wasn't allocated when system is under some pressure (page_is_pfmemalloc).
+ * 2) belongs to pool's numa node (pool->p.nid).
+ * 3) refcount is 1 (owned by page pool).
*
* To update pool->p.nid users must call page_pool_update_nid.
*/
-static bool pool_page_reusable(struct page_pool *pool, struct page *page)
+static bool page_pool_keep_page(struct page_pool *pool, struct page *page)
{
- return !page_is_pfmemalloc(page) && page_to_nid(page) == pool->p.nid;
+ return !page_is_pfmemalloc(page) &&
+ page_to_nid(page) == pool->p.nid &&
+ page_ref_count(page) == 1;
}
void __page_pool_put_page(struct page_pool *pool,
@@ -300,22 +304,19 @@ void __page_pool_put_page(struct page_pool *pool,
/* This allocator is optimized for the XDP mode that uses
* one-frame-per-page, but have fallbacks that act like the
* regular page allocator APIs.
- *
- * refcnt == 1 means page_pool owns page, and can recycle it.
*/
- if (likely(page_ref_count(page) == 1 &&
- pool_page_reusable(pool, page))) {
+
+ if (likely(page_pool_keep_page(pool, page))) {
/* Read barrier done in page_ref_count / READ_ONCE */
if (allow_direct && in_serving_softirq())
- if (__page_pool_recycle_direct(page, pool))
+ if (__page_pool_recycle_into_cache(page, pool))
return;
- if (!__page_pool_recycle_into_ring(pool, page)) {
- /* Cache full, fallback to free pages */
- __page_pool_return_page(pool, page);
- }
- return;
+ if (__page_pool_recycle_into_ring(pool, page))
+ return;
+
+ /* Cache full, fallback to return pages */
}
/* Fallback/non-XDP mode: API user have elevated refcnt.
*
@@ -330,8 +331,7 @@ void __page_pool_put_page(struct page_pool *pool,
* doing refcnt based recycle tricks, meaning another process
* will be invoking put_page.
*/
- __page_pool_clean_page(pool, page);
- put_page(page);
+ __page_pool_return_page(pool, page);
}
EXPORT_SYMBOL(__page_pool_put_page);
--
2.21.0
