bpf_prog_assoc_struct_ops() and bpf_prog_disassoc_struct_ops() update prog->aux->st_ops_assoc under st_ops_assoc_mutex, and the latter puts the map of a non-struct_ops prog while holding it.
A struct_ops map is about to dissociate its progs from map_free_pre_rcu(), which bpf_map_put() calls in whatever context drops the last reference, including softirq, e.g. bpf_struct_ops_put() from tcp_cleanup_congestion_control(). A mutex can't be taken there, and the put of the last reference under the lock of one prog would take the lock of other progs. Replace the mutex with a spinlock taken with interrupts disabled, and put the map after dropping it. Signed-off-by: Qiliang Yuan <[email protected]> --- include/linux/bpf.h | 2 +- kernel/bpf/bpf_struct_ops.c | 19 ++++++++++--------- kernel/bpf/core.c | 3 +-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 54144372281cb..2a51acd34f9a9 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1960,7 +1960,7 @@ struct bpf_prog_aux { struct rcu_head rcu; }; struct bpf_stream *stream[2]; - struct mutex st_ops_assoc_mutex; + spinlock_t st_ops_assoc_lock; /* protects st_ops_assoc */ struct bpf_map __rcu *st_ops_assoc; }; diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 1178acd72296e..a1ae30b145a03 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1541,10 +1541,10 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map) { struct bpf_map *st_ops_assoc; - guard(mutex)(&prog->aux->st_ops_assoc_mutex); + guard(spinlock_irqsave)(&prog->aux->st_ops_assoc_lock); st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc, - lockdep_is_held(&prog->aux->st_ops_assoc_mutex)); + lockdep_is_held(&prog->aux->st_ops_assoc_lock)); if (st_ops_assoc && st_ops_assoc == map) return 0; @@ -1569,19 +1569,20 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map) void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog) { + struct bpf_prog_aux *aux = prog->aux; struct bpf_map *st_ops_assoc; - guard(mutex)(&prog->aux->st_ops_assoc_mutex); + scoped_guard(spinlock_irqsave, &aux->st_ops_assoc_lock) { + st_ops_assoc = rcu_dereference_protected(aux->st_ops_assoc, + lockdep_is_held(&aux->st_ops_assoc_lock)); + if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON) + return; - st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc, - lockdep_is_held(&prog->aux->st_ops_assoc_mutex)); - if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON) - return; + RCU_INIT_POINTER(aux->st_ops_assoc, NULL); + } if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) bpf_map_put(st_ops_assoc); - - RCU_INIT_POINTER(prog->aux->st_ops_assoc, NULL); } /* diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 05c89396119ad..fd4e4dc4acc8c 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -141,7 +141,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag mutex_init(&fp->aux->used_maps_mutex); mutex_init(&fp->aux->ext_mutex); mutex_init(&fp->aux->dst_mutex); - mutex_init(&fp->aux->st_ops_assoc_mutex); + spin_lock_init(&fp->aux->st_ops_assoc_lock); return fp; } @@ -291,7 +291,6 @@ void __bpf_prog_free(struct bpf_prog *fp) #endif mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); - mutex_destroy(&fp->aux->st_ops_assoc_mutex); kfree(fp->aux->poke_tab); kfree(fp->aux); } -- 2.43.0
