rt_flush_dev() currently walks every per-CPU uncached route list for each device being removed. This repeatedly examines unrelated routes and makes teardown increasingly expensive as the number of devices grows.
Replace each per-CPU list with a hash table keyed by the route's netdevice. Keep the owning-list pointer in dst_entry so route removal remains unchanged, while device teardown only walks the matching bucket on each CPU. Hash collisions are filtered by the existing device comparison. The table has 2^CONFIG_IP_UNCACHED_ROUTE_HASH_BITS buckets and defaults to 64. Larger values shorten each bucket, but every additional bit doubles the per-CPU memory used by the table. The default costs approximately 1.5 KiB per possible CPU on x86-64. Signed-off-by: Chris J Arges <[email protected]> --- net/ipv4/Kconfig | 13 +++++++++++++ net/ipv4/route.c | 36 +++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index 301b47660305..7d40ca22d2b2 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -103,6 +103,19 @@ config IP_ROUTE_VERBOSE config IP_ROUTE_CLASSID bool +config IP_UNCACHED_ROUTE_HASH_BITS + int "IPv4 uncached route hash bits" + range 1 10 + default 6 + help + This option sets the number of buckets used in the IPv4 uncached + route hash table to 2^IP_UNCACHED_ROUTE_HASH_BITS buckets. The + allowed values select between 2 and 1024 buckets. Larger values + reduce collisions, but each additional bit doubles the per-CPU + memory used by the table. + + If unsure, use the default of 6 bits (64 buckets). + config IP_PNP bool "IP: kernel level autoconfiguration" help diff --git a/net/ipv4/route.c b/net/ipv4/route.c index d7da2f1acbb5..e28e2140cf62 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -74,6 +74,7 @@ #include <linux/init.h> #include <linux/skbuff.h> #include <linux/inetdevice.h> +#include <linux/hash.h> #include <linux/igmp.h> #include <linux/pkt_sched.h> #include <linux/mroute.h> @@ -1552,11 +1553,21 @@ struct uncached_list { struct list_head head; }; -static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list); +#define RT_UNCACHED_HASH_SIZE BIT(CONFIG_IP_UNCACHED_ROUTE_HASH_BITS) + +struct uncached_table { + struct uncached_list buckets[RT_UNCACHED_HASH_SIZE]; +}; + +static DEFINE_PER_CPU_ALIGNED(struct uncached_table, rt_uncached_table); void rt_add_uncached_list(struct rtable *rt) { - struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list); + struct uncached_table *table = raw_cpu_ptr(&rt_uncached_table); + struct uncached_list *ul; + + ul = &table->buckets[hash_ptr(dst_dev(&rt->dst), + CONFIG_IP_UNCACHED_ROUTE_HASH_BITS)]; rt->dst.rt_uncached_list = ul; @@ -1588,14 +1599,19 @@ void rt_flush_dev(struct net_device *dev) int cpu; for_each_possible_cpu(cpu) { - struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); + struct uncached_table *table; + struct uncached_list *ul; + + table = per_cpu_ptr(&rt_uncached_table, cpu); + ul = &table->buckets[hash_ptr(dev, + CONFIG_IP_UNCACHED_ROUTE_HASH_BITS)]; if (list_empty(&ul->head)) continue; spin_lock_bh(&ul->lock); list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { - if (rt->dst.dev != dev) + if (dst_dev(&rt->dst) != dev) continue; rcu_assign_pointer(rt->dst.dev_rcu, blackhole_netdev); netdev_ref_replace(dev, blackhole_netdev, @@ -3771,10 +3787,16 @@ int __init ip_rt_init(void) ip_tstamps = idents_hash + (ip_idents_mask + 1) * sizeof(*ip_idents); for_each_possible_cpu(cpu) { - struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); + struct uncached_table *table; + int bucket; + + table = per_cpu_ptr(&rt_uncached_table, cpu); + for (bucket = 0; bucket < RT_UNCACHED_HASH_SIZE; bucket++) { + struct uncached_list *ul = &table->buckets[bucket]; - INIT_LIST_HEAD(&ul->head); - spin_lock_init(&ul->lock); + INIT_LIST_HEAD(&ul->head); + spin_lock_init(&ul->lock); + } } #ifdef CONFIG_IP_ROUTE_CLASSID ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct)); -- 2.43.0
