Each received packet updates peer credit and calls sk_write_space() when send space is available. A writer cannot use the newly advertised credit until the socket lock is released, so repeated callbacks within one lock batch cannot let it make progress sooner.
For the callback installed by sock_init_data(), record one pending write-space notification and deliver it through the saved default callback before release_sock(). Save that callback when the AF_VSOCK socket is created because it is not visible to virtio_transport_common when built as a module. Keep only sockets using the saved callback eligible for batching. Sockets whose callback has been replaced, including sockmap sockets, retain their existing per-packet notification behavior. The psock marker checked by the batch admission path also excludes a socket before sockmap replaces this callback. Set the batch socket before processing its first packet so a batch ending on that packet cannot lose the notification. Packets outside the eligible STREAM/RW batch path retain per-packet notification behavior. The 64-packet and 64KB limits cap the packets whose notifications can be coalesced. Signed-off-by: Jia Jia <[email protected]> --- include/linux/virtio_vsock.h | 1 + include/net/af_vsock.h | 2 ++ net/vmw_vsock/af_vsock.c | 1 + net/vmw_vsock/virtio_transport_common.c | 42 ++++++++++++++++++------- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h index 26dde6909c4ed..bf077ee0ded88 100644 --- a/include/linux/virtio_vsock.h +++ b/include/linux/virtio_vsock.h @@ -287,6 +287,7 @@ struct virtio_transport_rx_batch { struct sock *sk; unsigned int pkts; size_t bytes; + bool write_space_pending; }; void virtio_transport_recv_pkt_batch(struct virtio_transport *t, diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h index 5549298c1ec6b..9d8ae62209eda 100644 --- a/include/net/af_vsock.h +++ b/include/net/af_vsock.h @@ -63,6 +63,8 @@ struct vsock_sock { u32 peer_shutdown; bool sent_request; bool ignore_connecting_rst; + /* Initial callback, used to identify replacements. */ + void (*default_write_space)(struct sock *sk); /* Protected by lock_sock(sk) */ u64 buffer_size; diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index 44cee74519555..fe7f49da6c4bc 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -958,6 +958,7 @@ static struct sock *__vsock_create(struct net *net, sk->sk_type = type; vsk = vsock_sk(sk); + vsk->default_write_space = sk->sk_write_space; vsock_addr_init(&vsk->local_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY); vsock_addr_init(&vsk->remote_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY); diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index bfe40ecf0b2f0..d6ebfde880db6 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -1820,6 +1820,7 @@ virtio_transport_recv_pkt_batchable(struct virtio_transport *t, return sk->sk_state == TCP_ESTABLISHED && sk->sk_type == SOCK_STREAM && READ_ONCE(sk->sk_prot) == sk->sk_prot_creator && + READ_ONCE(sk->sk_write_space) == vsk->default_write_space && !sock_flag(sk, SOCK_DONE) && vsk->transport == &t->transport && vvs && virtio_transport_rx_skb_has_headroom(vvs) && @@ -1836,7 +1837,8 @@ virtio_transport_recv_pkt_locked(struct virtio_transport *t, struct sk_buff *skb, struct sock *sk, const struct sockaddr_vm *src, const struct sockaddr_vm *dst, - struct net *net, bool *batchable) + struct net *net, bool *batchable, + struct virtio_transport_rx_batch *batch) { struct vsock_sock *vsk = vsock_sk(sk); bool space_available; @@ -1860,8 +1862,12 @@ virtio_transport_recv_pkt_locked(struct virtio_transport *t, if (vsk->local_addr.svm_cid != VMADDR_CID_ANY) vsk->local_addr.svm_cid = dst->svm_cid; - if (space_available) - sk->sk_write_space(sk); + if (space_available) { + if (batch) + batch->write_space_pending = true; + else + sk->sk_write_space(sk); + } switch (sk->sk_state) { case TCP_LISTEN: @@ -1929,7 +1935,7 @@ void virtio_transport_recv_pkt(struct virtio_transport *t, lock_sock(sk); free_pkt = virtio_transport_recv_pkt_locked(t, skb, sk, &src, &dst, - net, NULL); + net, NULL, NULL); release_sock(sk); /* Release refcnt obtained when we fetched this socket out of the @@ -1954,15 +1960,20 @@ EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); */ void virtio_transport_rx_batch_finish(struct virtio_transport_rx_batch *batch) { + bool write_space_pending = batch->write_space_pending; struct sock *sk = batch->sk; batch->sk = NULL; batch->pkts = 0; batch->bytes = 0; + batch->write_space_pending = false; if (!sk) return; + if (write_space_pending) + vsock_sk(sk)->default_write_space(sk); + release_sock(sk); sock_put(sk); } @@ -2018,7 +2029,8 @@ void virtio_transport_recv_pkt_batch(struct virtio_transport *t, free_pkt = virtio_transport_recv_pkt_locked(t, skb, sk, &src, &dst, net, - &batchable); + &batchable, + batch); if (!batchable) virtio_transport_rx_batch_finish(batch); if (free_pkt) @@ -2044,24 +2056,32 @@ void virtio_transport_recv_pkt_batch(struct virtio_transport *t, lock_sock(sk); /* - * Sockmap removal restores the native protocol under sk_callback_lock. - * Serialize this initial eligibility check with that update. + * Sockmap teardown restores callbacks under sk_callback_lock. Keep the + * initial eligibility check consistent with that update. */ read_lock_bh(&sk->sk_callback_lock); start_batch = virtio_transport_recv_pkt_batchable(t, sk); read_unlock_bh(&sk->sk_callback_lock); + if (start_batch) + batch->sk = sk; + free_pkt = virtio_transport_recv_pkt_locked(t, skb, sk, &src, &dst, net, - start_batch ? &batchable : NULL); + start_batch ? &batchable : NULL, + start_batch ? batch : NULL); if (start_batch && batchable) { /* Keep the lookup reference until the batch is released. */ - batch->sk = sk; return; } - release_sock(sk); - sock_put(sk); + if (start_batch) { + virtio_transport_rx_batch_finish(batch); + } else { + release_sock(sk); + sock_put(sk); + } + if (free_pkt) kfree_skb(skb); } -- 2.53.0
