October 2026 - dev

[PATCH v2 0/7] Add vhost-net kernel support to pasta
by aerosouund 16 Oct '26

16 Oct '26

From: Ammar Yasser <aerosound161(a)gmail.com> This work is based on the original v2 RFC by Eugenio Perez: Link: https://archives.passt.top/passt-dev/20250709174748.3514693-1-eperezma@redh… It is the first finished patch series for vhost-kernel support after a third RFC was submitted: Link: https://archives.passt.top/passt-dev/20260802132155.870796-1-aerosound161@g… vhost-net is a kernel device that allows reading and writing packets to a tap device using virtqueues instead of read(2), write(2), and their vectored counterparts. Frames are handed over through a shared descriptor ring, so several of them can be passed to the kernel at once rather than issuing a syscall per frame. The changes reliably yield a roughly 2x speedup on the TX path (container to host). Numbers were obtained through iperf3 with the following methodology and results: Pasta command: ./pasta -f --runas 0:0 --map-guest-addr 192.168.0.5 --vhost-kernel on --config-net 2729785 Host: iperf3 -s1J Container: iperf3 -c 192.168.0.5 -t 10 -P4 -l 1M -w 32M -i1 With --vhost-kernel on: [SUM] 0.00-10.00 sec 1.36 GBytes 1.17 Gbits/sec 3643 sender [SUM] 0.00-10.00 sec 1.16 GBytes 999 Mbits/sec receiver With --vhost-kernel off: [SUM] 0.00-10.00 sec 700 MBytes 587 Mbits/sec 3556 sender [SUM] 0.00-10.10 sec 495 MBytes 411 Mbits/sec receiver v2: - removed all use of epoll_ctl in vhost_setup_eventfds in favor of epoll_add - prevent all functions that register memory regions into the vhost_memory object from overflowing - remove the unions with flexible array members in vring_avail_pasta and vring_used_pasta in favor of hardcoding the ring array to have VHOST_NDESCS entries to avoid having to ignore Wpedantic - remove the && operator with TCP_CURR_IDX when computing no_csum - make desc_id in tx_reap a uint32_t - don't make vhost-kernel a required argument Ammar Yasser (7): tap: Move the tap_hdr file to a separate file conf: Add context fields, epoll types and --vhost-kernel flag to pasta virtio: Add the pasta vhost-net interface and implementation tap: Implement the pasta vhost-net from-guest path tap, tcp, udp: Prepare the to-guest path for vhost-net tap: Implement the pasta vhost-net to-guest path tap/tcp: Replace tcp_payload_used with a ring buffer style index Makefile | 2 +- conf.c | 19 +++ epoll_type.h | 4 + passt.c | 9 ++ passt.h | 79 ++++++++++- tap.c | 368 +++++++++++++++++++++++++++++++++++++++++++++------ tap.h | 47 ++----- tap_hdr.h | 23 ++++ tcp_buf.c | 149 ++++++++++++++++----- tcp_buf.h | 4 + udp.c | 47 +++++-- udp.h | 3 + util.c | 18 +++ util.h | 2 + vhost.c | 295 +++++++++++++++++++++++++++++++++++++++++ vhost.h | 91 +++++++++++++ 16 files changed, 1036 insertions(+), 124 deletions(-) create mode 100644 tap_hdr.h create mode 100644 vhost.c create mode 100644 vhost.h -- 2.39.5 (Apple Git-154)

1 2

[PATCH v2 1/7] tap: Move the tap_hdr file to a separate file
by aerosouund 16 Oct '26

16 Oct '26

From: Ammar Yasser <aerosound161(a)gmail.com> It was defined in tap.h, put it on its own so that future callers won't need to depend on all definitions in tap.h. Also turn it into a union of a vnet_len and virtio_net_mrg_rxbuf because for vhost acceleration the frames will have this virtio net header prepended to them. Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- tap.h | 9 +-------- tap_hdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 tap_hdr.h diff --git a/tap.h b/tap.h index b335933..1625975 100644 --- a/tap.h +++ b/tap.h @@ -10,6 +10,7 @@ #include <stdint.h> #include "passt.h" +#include "tap_hdr.h" /** L2_MAX_LEN_PASTA - Maximum frame length for pasta mode (with L2 header) * @@ -38,14 +39,6 @@ struct udphdr; -/** - * struct tap_hdr - tap backend specific headers - * @vnet_len: Frame length (for qemu socket transport) - */ -struct tap_hdr { - uint32_t vnet_len; -} __attribute__((packed)); - /** * tap_hdr_iov() - struct iovec for a tap header * @c: Execution context diff --git a/tap_hdr.h b/tap_hdr.h new file mode 100644 index 0000000..4156f44 --- /dev/null +++ b/tap_hdr.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright (c) 2021 Red Hat GmbH + * Author: David Gibson <david(a)gibson.dropbear.id.au> + */ + +#ifndef TAP_HDR_H +#define TAP_HDR_H + +#include <stdint.h> +#include <linux/virtio_net.h> + +/** + * struct tap_hdr - tap backend specific headers + * @vnet_len: Frame length (for qemu socket transport) + */ +struct tap_hdr { + union { + uint32_t vnet_len; + struct virtio_net_hdr_mrg_rxbuf hdr; + }; +}; + +#endif /* TAP_HDR_H */ -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 0/7] Add vhost-net kernel support to pasta
by aerosouund 16 Oct '26

16 Oct '26

From: Ammar Yasser <aerosound161(a)gmail.com> This work is based on the original v2 RFC by Eugenio Perez: Link: https://archives.passt.top/passt-dev/20250709174748.3514693-1-eperezma@redh… It is the first finished patch series for vhost-kernel support after a third RFC was submitted: Link: https://archives.passt.top/passt-dev/20260802132155.870796-1-aerosound161@g… vhost-net is a kernel device that allows reading and writing packets to a tap device using virtqueues instead of read(2), write(2), and their vectored counterparts. Frames are handed over through a shared descriptor ring, so several of them can be passed to the kernel at once rather than issuing a syscall per frame. The changes reliably yield a roughly 2x speedup on the TX path (container to host). Numbers were obtained through iperf3 with the following methodology and results: Pasta command: ./pasta -f --runas 0:0 --map-guest-addr 192.168.0.5 --vhost-kernel on --config-net 2729785 Host: iperf3 -s1J Container: iperf3 -c 192.168.0.5 -t 10 -P4 -l 1M -w 32M -i1 With --vhost-kernel on: [SUM] 0.00-10.00 sec 1.36 GBytes 1.17 Gbits/sec 3643 sender [SUM] 0.00-10.00 sec 1.16 GBytes 999 Mbits/sec receiver With --vhost-kernel off: [SUM] 0.00-10.00 sec 700 MBytes 587 Mbits/sec 3556 sender [SUM] 0.00-10.10 sec 495 MBytes 411 Mbits/sec receiver v2: - removed all use of epoll_ctl in vhost_setup_eventfds in favor of epoll_add - prevent all functions that register memory regions into the vhost_memory object from overflowing - remove the unions with flexible array members in vring_avail_pasta and vring_used_pasta in favor of hardcoding the ring array to have VHOST_NDESCS entries to avoid having to ignore Wpedantic - remove the && operator with TCP_CURR_IDX when computing no_csum - make desc_id in tx_reap a uint32_t - don't make vhost-kernel a required argument Ammar Yasser (7): tap: Move the tap_hdr file to a separate file conf: Add context fields, epoll types and --vhost-kernel flag to pasta virtio: Add the pasta vhost-net interface and implementation tap: Implement the pasta vhost-net from-guest path tap, tcp, udp: Prepare the to-guest path for vhost-net tap: Implement the pasta vhost-net to-guest path tap/tcp: Replace tcp_payload_used with a ring buffer style index Makefile | 2 +- conf.c | 19 +++ epoll_type.h | 4 + passt.c | 9 ++ passt.h | 79 ++++++++++- tap.c | 368 +++++++++++++++++++++++++++++++++++++++++++++------ tap.h | 47 ++----- tap_hdr.h | 23 ++++ tcp_buf.c | 149 ++++++++++++++++----- tcp_buf.h | 4 + udp.c | 47 +++++-- udp.h | 3 + util.c | 18 +++ util.h | 2 + vhost.c | 295 +++++++++++++++++++++++++++++++++++++++++ vhost.h | 91 +++++++++++++ 16 files changed, 1036 insertions(+), 124 deletions(-) create mode 100644 tap_hdr.h create mode 100644 vhost.c create mode 100644 vhost.h -- 2.39.5 (Apple Git-154) From 3b64b6ad029c1b6dde6a8de36fc0a035184673fe Mon Sep 17 00:00:00 2001 From: Ammar Yasser <aerosound161(a)gmail.com> Date: Sun, 2 Aug 2026 12:43:13 +0000 Subject: [PATCH v2 1/7] tap: Move the tap_hdr file to a separate file To: passt-dev(a)passt.top Cc: =?UTF-8?q?Eugenio=20P=C3=A9rez?= <eperezma(a)redhat.com> It was defined in tap.h, put it on its own so that future callers won't need to depend on all definitions in tap.h. Also turn it into a union of a vnet_len and virtio_net_mrg_rxbuf because for vhost acceleration the frames will have this virtio net header prepended to them. Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- tap.h | 9 +-------- tap_hdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 tap_hdr.h diff --git a/tap.h b/tap.h index b335933..1625975 100644 --- a/tap.h +++ b/tap.h @@ -10,6 +10,7 @@ #include <stdint.h> #include "passt.h" +#include "tap_hdr.h" /** L2_MAX_LEN_PASTA - Maximum frame length for pasta mode (with L2 header) * @@ -38,14 +39,6 @@ struct udphdr; -/** - * struct tap_hdr - tap backend specific headers - * @vnet_len: Frame length (for qemu socket transport) - */ -struct tap_hdr { - uint32_t vnet_len; -} __attribute__((packed)); - /** * tap_hdr_iov() - struct iovec for a tap header * @c: Execution context diff --git a/tap_hdr.h b/tap_hdr.h new file mode 100644 index 0000000..4156f44 --- /dev/null +++ b/tap_hdr.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright (c) 2021 Red Hat GmbH + * Author: David Gibson <david(a)gibson.dropbear.id.au> + */ + +#ifndef TAP_HDR_H +#define TAP_HDR_H + +#include <stdint.h> +#include <linux/virtio_net.h> + +/** + * struct tap_hdr - tap backend specific headers + * @vnet_len: Frame length (for qemu socket transport) + */ +struct tap_hdr { + union { + uint32_t vnet_len; + struct virtio_net_hdr_mrg_rxbuf hdr; + }; +}; + +#endif /* TAP_HDR_H */ -- 2.39.5 (Apple Git-154) From 4a48509734c572062a13e956e5943f6ae6781801 Mon Sep 17 00:00:00 2001 From: Ammar Yasser <aerosound161(a)gmail.com> Date: Thu, 3 Sep 2026 20:57:57 +0000 Subject: [PATCH v2 2/7] conf: Add context fields, epoll types and --vhost-kernel flag to pasta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To: passt-dev(a)passt.top Cc: =?UTF-8?q?Eugenio=20P=C3=A9rez?= <eperezma(a)redhat.com> --vhost-kernel controls whether to use vhost-net acceleration for pasta. It takes one of three modes: "on" requires acceleration and fails if it isn't available, "off" never uses it, and "auto" uses it if it's available and falls back to plain tap operation otherwise. "auto" is the default, so an unmodified invocation picks up acceleration wherever the kernel offers it, and behaves as before where it doesn't. EPOLL_TYPE_VHOST_CALL means the kernel wants to notify us about data it has written. EPOLL_TYPE_VHOST_ERROR means the kernel encountered an internal error on vhost and wants to tell us something went wrong. Collect the vhost state in a single struct vhost_ctx with the following fields: - mode: which of the three modes above was requested - fd: file descriptor for /dev/vhost-net, which doubles as our indication that setup succeeded - features: virtio feature bits negotiated with vhost-net - vq[]: kick, call and err eventfds for both queues Modify tap_hdr_iov to create an iov with the size of a virtio_net_mrg_rxbuf if vhost initialization has been successful (we are in mode 'on' or 'auto' and we didn't need the tap fallback) Move the L2_MAX_LEN_XX constants from tap.h to passt.h to avoid the circular dependency between both files. VHOST_DESC_BYTES is defined as the maximum l2 frame size + the size of the virtio net header, if the length constant remains in tap.h passt.h would need to include it, but tap.h needs to include passt.h to get the definition of `struct ctx`. Moving the length constants to passt.h makes it free of dependency on tap.h Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- conf.c | 19 +++++++++++++ epoll_type.h | 4 +++ passt.c | 3 ++ passt.h | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++-- tap.h | 36 ++++++------------------ 5 files changed, 112 insertions(+), 29 deletions(-) diff --git a/conf.c b/conf.c index 149a4c7..fe578fc 100644 --- a/conf.c +++ b/conf.c @@ -736,6 +736,9 @@ pasta_opts: " default: auto\n" " --host-lo-to-ns-lo Translate host-loopback forwards to\n" " namespace loopback\n" + " --vhost-kernel[=MODE] Use vhost-kernel acceleration\n" + " MODE is \"on\", \"off\" or \"auto\"\n" + " default: auto\n" " --userns NSPATH Target user namespace to join\n" " --netns PATH|NAME Target network namespace to join\n" " --netns-only Don't join existing user namespace\n" @@ -767,6 +770,7 @@ enum passt_modes conf_mode(int argc, char *argv[]) int vhost_user = 0; const struct option optvu[] = { {"vhost-user", no_argument, &vhost_user, 1 }, + {"vhost-kernel", optional_argument, NULL, 0 }, { 0 }, }; char argv0[PATH_MAX], *basearg0; @@ -1317,6 +1321,7 @@ void conf(struct ctx *c, int argc, char **argv) {"conf-path", required_argument, NULL, 'c' }, {"chroot-fallback", no_argument, NULL, 32 }, {"no-pidns", no_argument, NULL, 33 }, + {"vhost-kernel", optional_argument, NULL, 34 }, { 0 }, }; const char *optstring = "+dqfel:hs:c:F:I:p:P:m:a:n:M:g:i:o:D:S:H:461t:u:T:U:"; @@ -1566,6 +1571,20 @@ void conf(struct ctx *c, int argc, char **argv) die("--no-pidns is for pasta mode only"); c->no_pidns = true; + break; + case 34: + if (c->mode != MODE_PASTA) + die("--vhost-kernel is for pasta mode only"); + + if (!optarg || !strcmp(optarg, "auto")) + c->vhost.mode = VHOST_MODE_AUTO; + else if (!strcmp(optarg, "on")) + c->vhost.mode = VHOST_MODE_ON; + else if (!strcmp(optarg, "off")) + c->vhost.mode = VHOST_MODE_OFF; + else + die("Invalid vhost-kernel mode: %s", optarg); + break; case 'd': c->debug = 1; diff --git a/epoll_type.h b/epoll_type.h index 061325a..e3206d1 100644 --- a/epoll_type.h +++ b/epoll_type.h @@ -50,6 +50,10 @@ enum epoll_type { EPOLL_TYPE_CONF_LISTEN, /* Configuration socket */ EPOLL_TYPE_CONF, + /* vhost-kernel call socket */ + EPOLL_TYPE_VHOST_CALL, + /* vhost-kernel error socket */ + EPOLL_TYPE_VHOST_ERROR, EPOLL_NUM_TYPES, }; diff --git a/passt.c b/passt.c index 5054551..6002231 100644 --- a/passt.c +++ b/passt.c @@ -65,6 +65,7 @@ char pkt_buf[PKT_BUF_BYTES] __attribute__ ((aligned(PAGE_SIZE))); struct ctx passt_ctx = { .pidfile_fd = -1, .fd_tap = -1, + .vhost = { .fd = -1 }, .fd_tap_listen = -1, .fd_control_listen = -1, .fd_repair_listen = -1, @@ -92,6 +93,8 @@ char *epoll_type_str[] = { [EPOLL_TYPE_NL_NEIGH] = "netlink neighbour notifier socket", [EPOLL_TYPE_CONF_LISTEN] = "configuration listening socket", [EPOLL_TYPE_CONF] = "configuration socket", + [EPOLL_TYPE_VHOST_CALL] = "vhost-kernel call socket", + [EPOLL_TYPE_VHOST_ERROR] = "vhost-kernel error socket", }; static_assert(ARRAY_SIZE(epoll_type_str) == EPOLL_NUM_TYPES, "epoll_type_str[] doesn't match enum epoll_type"); diff --git a/passt.h b/passt.h index afd8e9f..be68002 100644 --- a/passt.h +++ b/passt.h @@ -13,8 +13,11 @@ union epoll_ref; #include <stdbool.h> #include <assert.h> +#include <limits.h> #include <sys/epoll.h> +#include <linux/virtio_net.h> + #include "pif.h" #include "packet.h" #include "siphash.h" @@ -35,8 +38,43 @@ union epoll_ref; #define MAC_OUR_LAA \ ((uint8_t [ETH_ALEN]){0x9a, 0x55, 0x9a, 0x55, 0x9a, 0x55}) -/* Large enough for ~128 maximum size frames */ -#define PKT_BUF_BYTES (8UL << 20) + +/** L2_MAX_LEN_PASTA - Maximum frame length for pasta mode (with L2 header) + * + * The kernel tuntap device imposes a maximum frame size of 65535 including + * 'hard_header_len' (14 bytes for L2 Ethernet in the case of "tap" mode). + */ +#define L2_MAX_LEN_PASTA USHRT_MAX + +/** L2_MAX_LEN_PASST - Maximum frame length for passt mode (with L2 header) + * + * The only structural limit the QEMU socket protocol imposes on frames is + * (2^32-1) bytes, but that would be ludicrously long in practice. For now, + * limit it somewhat arbitrarily to 65535 bytes. FIXME: Work out an appropriate + * limit with more precision. + */ +#define L2_MAX_LEN_PASST USHRT_MAX + +/** L2_MAX_LEN_VU - Maximum frame length for vhost-user mode (with L2 header) + * + * vhost-user allows multiple buffers per frame, each of which can be quite + * large, so the inherent frame size limit is rather large. Much larger than is + * actually useful for IP. For now limit arbitrarily to 65535 bytes. FIXME: + * Work out an appropriate limit with more precision. + */ +#define L2_MAX_LEN_VU USHRT_MAX + +/* Number of descriptors in each vhost-net virtqueue */ +#define VHOST_NDESCS 128 + +/* Bytes of pkt_buf backing one from-guest descriptor: a maximum size frame + * plus the virtio-net header the kernel writes in front of it + */ +#define VHOST_DESC_BYTES (L2_MAX_LEN_PASTA + \ + sizeof(struct virtio_net_hdr_mrg_rxbuf)) + +/* One maximum size frame per vhost-net descriptor */ +#define PKT_BUF_BYTES (VHOST_NDESCS * VHOST_DESC_BYTES) extern char pkt_buf [PKT_BUF_BYTES]; @@ -156,6 +194,40 @@ struct ip6_ctx { #include <netinet/if_ether.h> +/** + * enum vhost_mode - Whether to use vhost-kernel acceleration + * @VHOST_MODE_AUTO: Use it if it's available, fall back to plain tap if not + * @VHOST_MODE_ON: Require it, fail if it's not available + * @VHOST_MODE_OFF: Never use it + */ +enum vhost_mode { + VHOST_MODE_AUTO = 0, + VHOST_MODE_ON, + VHOST_MODE_OFF, +}; + +/** + * struct vhost_ctx - Execution context for vhost-kernel acceleration + * @mode: Whether to use acceleration at all, see enum vhost_mode + * @fd: File descriptor for /dev/vhost-net, -1 if not set up + * @features: virtio feature bits negotiated with vhost-net + * @vq: Per-virtqueue eventfds ([0] is from-guest, [1] is to-guest) + * @vq.kick_fd: Written by us, to tell the kernel we queued something + * @vq.call_fd: Written by the kernel, to tell us it queued something + * @vq.err_fd: Written by the kernel on an internal vhost error + */ +struct vhost_ctx { + enum vhost_mode mode; + int fd; + uint64_t features; + + struct { + int kick_fd; + int call_fd; + int err_fd; + } vq[2]; +}; + /** * struct ctx - Execution context * @mode: Operation mode, qemu/UNIX domain socket or namespace/tap @@ -185,6 +257,7 @@ struct ip6_ctx { * @our_tap_mac: Pasta/passt's MAC on the tap link * @guest_mac: MAC address of guest or namespace, seen or configured * @hash_secret: 128-bit secret for siphash functions + * @vhost: vhost-kernel acceleration context, pasta mode only * @ifi4: Template interface for IPv4, -1: none, 0: IPv4 disabled * @ip4: IPv4 configuration * @dns_search: DNS search list @@ -265,6 +338,8 @@ struct ctx { uint64_t hash_secret[2]; + struct vhost_ctx vhost; + int ifi4; struct ip4_ctx ip4; diff --git a/tap.h b/tap.h index 1625975..f8c30e2 100644 --- a/tap.h +++ b/tap.h @@ -12,31 +12,6 @@ #include "passt.h" #include "tap_hdr.h" -/** L2_MAX_LEN_PASTA - Maximum frame length for pasta mode (with L2 header) - * - * The kernel tuntap device imposes a maximum frame size of 65535 including - * 'hard_header_len' (14 bytes for L2 Ethernet in the case of "tap" mode). - */ -#define L2_MAX_LEN_PASTA USHRT_MAX - -/** L2_MAX_LEN_PASST - Maximum frame length for passt mode (with L2 header) - * - * The only structural limit the QEMU socket protocol imposes on frames is - * (2^32-1) bytes, but that would be ludicrously long in practice. For now, - * limit it somewhat arbitrarily to 65535 bytes. FIXME: Work out an appropriate - * limit with more precision. - */ -#define L2_MAX_LEN_PASST USHRT_MAX - -/** L2_MAX_LEN_VU - Maximum frame length for vhost-user mode (with L2 header) - * - * vhost-user allows multiple buffers per frame, each of which can be quite - * large, so the inherent frame size limit is rather large. Much larger than is - * actually useful for IP. For now limit arbitrarily to 65535 bytes. FIXME: - * Work out an appropriate limit with more precision. - */ -#define L2_MAX_LEN_VU USHRT_MAX - struct udphdr; /** @@ -50,10 +25,17 @@ struct udphdr; static inline struct iovec tap_hdr_iov(const struct ctx *c, struct tap_hdr *thdr) { - return (struct iovec){ + struct iovec ret = { .iov_base = thdr, - .iov_len = c->mode == MODE_PASST ? sizeof(*thdr) : 0, }; + + if (c->vhost.fd != -1) { + ret.iov_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); + } else { + ret.iov_len = c->mode == MODE_PASST ? sizeof(thdr->vnet_len) : 0; + } + + return ret; } /** -- 2.39.5 (Apple Git-154) From cd8b50e7ffaf7232faf03b28bf61682db9237814 Mon Sep 17 00:00:00 2001 From: Ammar Yasser <aerosound161(a)gmail.com> Date: Thu, 3 Sep 2026 22:01:39 +0000 Subject: [PATCH v2 3/7] virtio: Add the pasta vhost-net interface and implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To: passt-dev(a)passt.top Cc: =?UTF-8?q?Eugenio=20P=C3=A9rez?= <eperezma(a)redhat.com> Add vhost.c and vhost.h which define the api and implementation for pasta vhost acceleration - vq_state/vhost_vq_state[]: our own view of each virtqueue, which the kernel doesn't track for us: descriptors ready to hand back, how far we've read the used ring, and the next descriptor to allocate - vring_desc, vring_avail_all, vring_used_all, vhost_memory: the rings and the region table shared with the kernel - vhost_setup_net: open /dev/vhost-net and negotiate VIRTIO_F_VERSION_1 and VHOST_NET_F_VIRTIO_NET_HDR. Failure is fatal only if --vhost-kernel is set to 'on'. - vhost_setup_eventfds: create, register and watch the call, kick and error eventfds for one queue - vhost_setup_memory_table: register the buffers the kernel may touch. pasta has no guest, so each region maps guest_phys_addr onto the identical userspace_addr, making GPA translation an identity mapping. This function uses the *_register_memory_regions functions created per protocol and for general purpose buffers. - vhost_set_vring: publish a vring's addresses, bind the tap fd as its backend and lay out its descriptors - vhost_rx_descriptor_handoff, vhost_kick: hand consumed descriptors back and notify the kernel unless it says it's already polling Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- Makefile | 2 +- tcp_buf.c | 22 +++- tcp_buf.h | 4 + udp.c | 20 ++++ udp.h | 3 + util.c | 18 ++++ util.h | 2 + vhost.c | 295 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ vhost.h | 91 +++++++++++++++++ 9 files changed, 455 insertions(+), 2 deletions(-) create mode 100644 vhost.c create mode 100644 vhost.h diff --git a/Makefile b/Makefile index f6c87b1..6f11e4b 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ PASST_SRCS = arch.c arp.c bitmap.c checksum.c conf.c dhcp.c dhcpv6.c \ isolation.c lineread.c log.c mld.c ndp.c netlink.c migrate.c packet.c \ parse.c passt.c pasta.c pcap.c pif.c repair.c serialise.c tap.c tcp.c \ tcp_buf.c tcp_splice.c tcp_vu.c udp.c udp_flow.c udp_vu.c util.c \ - vhost_user.c virtio.c vu_common.c + vhost.c vhost_user.c virtio.c vu_common.c PASST_REPAIR_SRCS = passt-repair.c PESTO_SRCS = pesto.c bitmap.c fwd_rule.c inany.c ip.c lineread.c parse.c \ serialise.c diff --git a/tcp_buf.c b/tcp_buf.c index 72c4541..dd80384 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -31,12 +31,14 @@ #include "inany.h" #include "tcp_conn.h" #include "tcp_internal.h" -#include "tcp_buf.h" +#include "vhost.h" #define TCP_FRAMES_MEM 128 #define TCP_FRAMES \ (c->mode == MODE_PASTA ? 1 : TCP_FRAMES_MEM) +#define TCP_NREGIONS 5 + /* Static buffers */ /* Ethernet header for IPv4 and IPv6 frames */ @@ -75,6 +77,24 @@ void tcp_update_l2_buf(const unsigned char *eth_d) eth_update_mac(&tcp_eth_hdr[i], eth_d, NULL); } +/** + * tcp_register_memory_region() - Register the TCP specific buffers into a memory + * struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void tcp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + TCP_NREGIONS) > N_VHOST_REGIONS) + die("tcp: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + TCP_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_payload_tap_hdr); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp4_payload_ip); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp6_payload_ip); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_payload); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_eth_hdr); +} + /** * tcp_sock_iov_init() - Initialise scatter-gather L2 buffers for IPv4 sockets * @c: Execution context diff --git a/tcp_buf.h b/tcp_buf.h index 5d31cea..5e341ed 100644 --- a/tcp_buf.h +++ b/tcp_buf.h @@ -6,11 +6,15 @@ #ifndef TCP_BUF_H #define TCP_BUF_H +union vhost_memory_u; +struct tcp_tap_conn; + void tcp_sock_iov_init(const struct ctx *c); void tcp_payload_flush(const struct ctx *c, const struct timespec *now); int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, uint32_t already_sent, const struct timespec *now); int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, const struct timespec *now); +void tcp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); #endif /*TCP_BUF_H */ diff --git a/udp.c b/udp.c index 505e554..a4355a1 100644 --- a/udp.c +++ b/udp.c @@ -118,6 +118,7 @@ #include "udp_internal.h" #include "udp_vu.h" #include "epoll_ctl.h" +#include "vhost.h" #define UDP_MAX_FRAMES 32 /* max # of frames to receive at once */ @@ -128,6 +129,8 @@ #define UDP_TIMEOUT_DEFAULT 30 /* s */ #define UDP_TIMEOUT_STREAM_DEFAULT 120 /* s */ +#define UDP_NREGIONS 5 + /* Maximum UDP data to be returned in ICMP messages */ #define ICMP4_MAX_DLEN 8 #define ICMP6_MAX_DLEN (IPV6_MIN_MTU \ @@ -448,6 +451,23 @@ static void udp_send_tap_icmp4(const struct ctx *c, tap_icmp4_send(c, saddr, eaddr, &msg, tap_omac, msglen); } +/** + * udp_register_memory_region() - Register the UDP specific buffers into a memory + * struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void udp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + UDP_NREGIONS) > N_VHOST_REGIONS) + die("udp: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + UDP_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_payload); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_eth_hdr); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_iov_recv); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_mh_recv); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_meta); +} /** * udp_send_tap_icmp6() - Construct and send ICMPv6 to local peer diff --git a/udp.h b/udp.h index b50283e..ebf1a4e 100644 --- a/udp.h +++ b/udp.h @@ -11,6 +11,8 @@ #include "fwd.h" +union vhost_memory_u; + void udp_listen_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, const struct timespec *now); void udp_sock_handler(const struct ctx *c, union epoll_ref ref, @@ -21,6 +23,7 @@ int udp_tap_handler(const struct ctx *c, uint8_t pif, const struct timespec *now); int udp_init(struct ctx *c); void udp_update_l2_buf(const unsigned char *eth_d); +void udp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); /** * struct udp_ctx - Execution context for UDP diff --git a/util.c b/util.c index ff1f19f..cafe0ec 100644 --- a/util.c +++ b/util.c @@ -33,10 +33,13 @@ #include "log.h" #include "pcap.h" #include "serialise.h" +#include "vhost.h" #ifdef HAS_GETRANDOM #include <sys/random.h> #endif +#define GENERAL_NREGIONS 2 + /* Zero-filled buffer to pad 802.3 frames, up to 60 (ETH_ZLEN) bytes */ uint8_t eth_pad[ETH_ZLEN] = { 0 }; @@ -453,6 +456,21 @@ int open_in_ns(const struct ctx *c, const char *path, int flags) return arg.fd; } +/** + * general_register_memory_region() - Register the general (non protocol specific) buffers + * into a memory struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void general_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + GENERAL_NREGIONS) > N_VHOST_REGIONS) + die("general: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + GENERAL_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(pkt_buf); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(eth_pad); +} + /** * pidfile_write() - Write PID to file, if requested to do so, and close it * @fd: Open PID file descriptor, closed on exit, -1 to skip writing it diff --git a/util.h b/util.h index ec2d9ac..11f6877 100644 --- a/util.h +++ b/util.h @@ -145,6 +145,7 @@ int do_clone(int (*fn)(void *), char *stack_area, size_t stack_size, int flags, struct ctx; union sockaddr_inany; +union vhost_memory_u; int sock_l4(const struct ctx *c, enum epoll_type type, const union sockaddr_inany *sa, const char *ifname); @@ -170,6 +171,7 @@ int write_remainder(int fd, const struct iovec *iov, size_t iovcnt, int read_remainder(int fd, const struct iovec *iov, size_t cnt, size_t skip); bool snprintf_check(char *str, size_t size, const char *format, ...); long clamped_scale(long x, long y, long lo, long hi, long f); +void general_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); /** * af_name() - Return name of an address family diff --git a/vhost.c b/vhost.c new file mode 100644 index 0000000..a69c4b5 --- /dev/null +++ b/vhost.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Copyright Red Hat + * Author: Ammar Yasser <aerosound161(a)gmail.com> + * + * vhost.c - vhost-net (vhost-kernel) acceleration for pasta mode + */ + +#include <errno.h> +#include <fcntl.h> +#include <inttypes.h> +#include <stdint.h> +#include <string.h> +#include <unistd.h> +#include <sys/eventfd.h> +#include <sys/ioctl.h> + +#include "util.h" +#include "passt.h" +#include "vhost.h" +#include "epoll_ctl.h" +#include "udp.h" +#include "tcp_buf.h" + +struct vq_state vhost_vq_state[2]; + +struct vring_desc_pasta vring_desc[2][VHOST_NDESCS] + __attribute__((aligned(PAGE_SIZE))); + +struct vring_avail_pasta vring_avail_all[2] + __attribute__((aligned(PAGE_SIZE))); +struct vring_used_pasta vring_used_all[2] + __attribute__((aligned(PAGE_SIZE))); + +union vhost_memory_u vhost_memory = { + .mem = { + .nregions = N_VHOST_REGIONS, + }, +}; + +/** + * vhost_setup_net() - Open and negotiate features on /dev/vhost-net + * @c: Execution context; c->vhost.fd and c->vhost.features are set + * on success, and left untouched on failure + * + * Failure here means vhost-net isn't usable, not that pasta can't run: the + * caller falls back to plain tap operation unless acceleration was required. + * That's why nothing in here is fatal. + * + * Return: 0 on success, -1 if vhost-net is unavailable or unusable + */ +int vhost_setup_net(struct ctx *c) +{ + const uint64_t req_features = (1ULL << VIRTIO_F_VERSION_1) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t features; + int vhost_fd; + + vhost_fd = open("/dev/vhost-net", O_RDWR | O_NONBLOCK | O_CLOEXEC); + if (vhost_fd < 0) { + debug_perror("Couldn't open /dev/vhost-net"); + return -1; + } + + if (ioctl(vhost_fd, VHOST_SET_OWNER, NULL) < 0) { + debug_perror("VHOST_SET_OWNER ioctl failed"); + goto close_fd; + } + + if (ioctl(vhost_fd, VHOST_GET_FEATURES, &features) < 0) { + debug_perror("VHOST_GET_FEATURES ioctl failed"); + goto close_fd; + } + + if ((features & req_features) != req_features) { + debug("vhost-net is missing features: 0x%016" PRIx64, + req_features & ~features); + goto close_fd; + } + + features = req_features; + if (ioctl(vhost_fd, VHOST_SET_FEATURES, &features) < 0) { + debug_perror("VHOST_SET_FEATURES ioctl failed"); + goto close_fd; + } + + c->vhost.features = features; + c->vhost.fd = vhost_fd; + + return 0; + +close_fd: + close(vhost_fd); + + return -1; +} + +/** + * vhost_setup_eventfds() - Set up one queue's eventfds and ring size + * @c: Execution context; c->vhost.fd must already be set + * @queue_idx: Index of the queue (vring) to configure + * + */ +void vhost_setup_eventfds(struct ctx *c, int queue_idx) +{ + struct vhost_vring_file vring_file = { .index = queue_idx }; + struct vhost_vring_state state = { .index = queue_idx }; + union epoll_ref ref = { 0 }; + int vhost_fd = c->vhost.fd; + int rc; + + state.num = VHOST_NDESCS; + ref.queue = queue_idx; + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost call eventfd, queue %d", + queue_idx); + ref.fd = vring_file.fd; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_CALL, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_CALL ioctl failed, queue %d", + queue_idx); + + ref.type = EPOLL_TYPE_VHOST_CALL; + rc = epoll_add(c->epollfd, EPOLLIN, ref); + if (rc < 0) + die_perror("Failed to watch vhost call eventfd, queue %d", + queue_idx); + c->vhost.vq[queue_idx].call_fd = vring_file.fd; + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost error eventfd, queue %d", + queue_idx); + + rc = ioctl(vhost_fd, VHOST_SET_VRING_ERR, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_ERR ioctl failed, queue %d", + queue_idx); + + ref.type = EPOLL_TYPE_VHOST_ERROR; + ref.fd = vring_file.fd; + rc = epoll_add(c->epollfd, EPOLLIN, ref); + if (rc < 0) + die_perror("Failed to watch vhost error eventfd, queue %d", + queue_idx); + c->vhost.vq[queue_idx].err_fd = vring_file.fd; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_NUM, &state); + if (rc < 0) + die_perror("VHOST_SET_VRING_NUM ioctl failed, queue %d", + queue_idx); + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost kick eventfd, queue %d", + queue_idx); + + rc = ioctl(vhost_fd, VHOST_SET_VRING_KICK, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_KICK ioctl failed, queue %d", + queue_idx); + + c->vhost.vq[queue_idx].kick_fd = vring_file.fd; + + vhost_vq_state[queue_idx].num_free = VHOST_NDESCS; +} + +/** + * vhost_setup_memory_table() - Register the GPA/HVA translation table + * @c: Execution context; c->vhost.fd must already be set + * + * vhost-net reads the addresses we put in descriptors as guest physical + * addresses, and translates them through this table before touching the + * memory they refer to. pasta has no guest and no second address space: + * the addresses we put there are our own virtual addresses. Every region + * below therefore sets guest_phys_addr equal to userspace_addr, so that + * GPA == HVA and the translation is an identity mapping. + * + * Return: 0 on success, -1 on error with errno set + */ +int vhost_setup_memory_table(struct ctx *c) { + size_t region_idx = 0; + + /* general purpose buffers */ + general_register_memory_regions(&vhost_memory, &region_idx); + + /* tcp specific buffers */ + tcp_register_memory_regions(&vhost_memory, &region_idx); + + /* udp specific buffers */ + udp_register_memory_regions(&vhost_memory, &region_idx); + + vhost_memory.mem.nregions = region_idx; + + return ioctl(c->vhost.fd, VHOST_SET_MEM_TABLE, &vhost_memory.mem); +} + +/** + * vhost_set_vring() - Register a vring's addresses and bind its backend + * @c: Execution context; c->vhost.fd must already be set + * @queue_idx: Index of the queue (vring) to configure + * @tap_fd: Tap fd to bind as this queue's backend + */ +void vhost_set_vring(struct ctx *c, int queue_idx, int tap_fd) +{ + int vhost_fd = c->vhost.fd; + struct vhost_vring_addr addr = { + .index = queue_idx, + .desc_user_addr = (unsigned long)vring_desc[queue_idx], + .avail_user_addr = (unsigned long)&vring_avail_all[queue_idx], + .used_user_addr = (unsigned long)&vring_used_all[queue_idx], + .log_guest_addr = (unsigned long)&vring_used_all[queue_idx], + }; + struct vhost_vring_file file = { + .index = queue_idx, + .fd = tap_fd, + }; + unsigned int i; + int rc; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_ADDR, &addr); + if (rc < 0) + die_perror("VHOST_SET_VRING_ADDR ioctl failed, queue %d", + queue_idx); + + if (queue_idx == VHOST_AVAIL_Q_IDX) { + for (i = 0; i < VHOST_NDESCS; ++i) { + vring_desc[VHOST_AVAIL_Q_IDX][i].addr = + (uintptr_t)pkt_buf + i * VHOST_DESC_BYTES; + vring_desc[VHOST_AVAIL_Q_IDX][i].len = VHOST_DESC_BYTES; + vring_desc[VHOST_AVAIL_Q_IDX][i].flags = + VRING_DESC_F_WRITE; + } + + for (i = 0; i < VHOST_NDESCS; ++i) + vring_avail_all[VHOST_AVAIL_Q_IDX].ring[i] = + htole16(i); + + vhost_rx_descriptor_handoff(c); + } + + if (queue_idx == VHOST_USED_Q_IDX) { + for (i = 0; i < (VHOST_NDESCS - 1); ++i) + vring_desc[VHOST_USED_Q_IDX][i].next = i + 1; + } + + rc = ioctl(vhost_fd, VHOST_NET_SET_BACKEND, &file); + if (rc < 0) + die_perror("VHOST_NET_SET_BACKEND ioctl failed, queue %d", + queue_idx); +} + +/** + * vhost_rx_descriptor_handoff() - Announce freed from-guest descriptors + * @c: Execution context + * + * Bumps avail.idx by the number of descriptors accumulated in + * vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free (from prior + * consume_one_rx_descriptor() calls), + * then resets the counter to zero. The kernel will see the new + * avail.idx and consume the freshly-available descriptors. + */ +void vhost_rx_descriptor_handoff(struct ctx *c) +{ + smp_wmb(); + + if (!vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free) + return; + + vring_avail_all[VHOST_AVAIL_Q_IDX].idx += + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free; + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free = 0; + vhost_kick(&vring_used_all[VHOST_AVAIL_Q_IDX], + c->vhost.vq[VHOST_AVAIL_Q_IDX].kick_fd); +} + +/** + * vhost_kick() - Notify the kernel that new descriptors are available + * @used: Used ring of the queue we're announcing on, checked to see + * whether the kernel's virtio thread is already reading + * descriptors and doesn't want to be notified + * @kick_fd: Kick eventfd of that same queue + */ +void vhost_kick(struct vring_used_pasta *used, int kick_fd) +{ + /* Ensure that the read of used->flags doesn't get reordered to be + * above the avail.idx update + */ + smp_mb(); + + if (!(used->flags & VRING_USED_F_NO_NOTIFY)) + eventfd_write(kick_fd, 1); +} diff --git a/vhost.h b/vhost.h new file mode 100644 index 0000000..bd555f9 --- /dev/null +++ b/vhost.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: Ammar Yasser <aerosound161(a)gmail.com> + * + * vhost.h - vhost-net (vhost-kernel) acceleration for pasta mode + */ + +#ifndef VHOST_H +#define VHOST_H + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> + +#include <linux/vhost.h> +#include <linux/vhost_types.h> +#include <linux/virtio_config.h> +#include <linux/virtio_net.h> +#include <linux/virtio_ring.h> + +#include "passt.h" + +#define VHOST_AVAIL_Q_IDX 0 +#define VHOST_USED_Q_IDX 1 + +/** + * struct vq_state - Per-virtqueue local descriptor tracking + * @num_free: Number of descriptors ready to be announced to the + * kernel via vhost_rx_descriptor_handoff() + * @last_used_idx: Number of used-ring entries consumed so far; + * lagging read cursor vs. vring_used->idx (the + * kernel's write cursor) + */ +extern struct vq_state { + uint16_t num_free; + uint16_t last_used_idx; + uint16_t next_free; +} vhost_vq_state[2]; + +struct vring_desc_pasta { + __virtio64 addr; + __virtio32 len; + __virtio16 flags; + __virtio16 next; +}; + +extern struct vring_desc_pasta vring_desc[2][VHOST_NDESCS]; + +struct vring_avail_pasta { + __virtio16 flags; + __virtio16 idx; + __virtio16 ring[VHOST_NDESCS]; +}; +static_assert(sizeof(struct vring_avail_pasta) == + offsetof(struct vring_avail, ring[VHOST_NDESCS]), + "vring_avail_pasta doesn't match the kernel avail ring layout"); +extern struct vring_avail_pasta vring_avail_all[2]; + +struct vring_used_pasta { + __virtio16 flags; + __virtio16 idx; + vring_used_elem_t ring[VHOST_NDESCS]; +}; +static_assert(sizeof(struct vring_used_pasta) == + offsetof(struct vring_used, ring[VHOST_NDESCS]), + "vring_used_pasta doesn't match the kernel used ring layout"); +extern struct vring_used_pasta vring_used_all[2]; + +#define N_VHOST_REGIONS 12 +union vhost_memory_u { + struct vhost_memory mem; + char buf[offsetof(struct vhost_memory, regions[N_VHOST_REGIONS])]; +}; +extern union vhost_memory_u vhost_memory; + +#define VHOST_MEMORY_REGION_PTR(addr, size) \ + (struct vhost_memory_region) { \ + .guest_phys_addr = (uintptr_t)addr, \ + .memory_size = size, \ + .userspace_addr = (uintptr_t)addr, \ + } +#define VHOST_MEMORY_REGION(buf) VHOST_MEMORY_REGION_PTR(&buf, sizeof(buf)) + +void vhost_set_vring(struct ctx *c, int queue_idx, int tap_fd); +int vhost_setup_memory_table(struct ctx *c); +int vhost_setup_net(struct ctx *c); +void vhost_setup_eventfds(struct ctx *c, int queue_idx); +void vhost_rx_descriptor_handoff(struct ctx *c); +void vhost_kick(struct vring_used_pasta *used, int kick_fd); + +#endif /* VHOST_H */ -- 2.39.5 (Apple Git-154) From 3ff090d18c33e9a5c131b8ffa581a272a00fc050 Mon Sep 17 00:00:00 2001 From: Ammar Yasser <aerosound161(a)gmail.com> Date: Fri, 4 Sep 2026 13:01:25 +0000 Subject: [PATCH v2 4/7] tap: Implement the pasta vhost-net from-guest path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To: passt-dev(a)passt.top Cc: =?UTF-8?q?Eugenio=20P=C3=A9rez?= <eperezma(a)redhat.com> Add tap_vhost_input(), called when the kernel signals on a call eventfd that it has written guest frames into the from-guest queue. It consumes descriptors with consume_one_rx_descriptor(), skips the virtio_net header on each, builds an iov_tail from what's left and queues it for processing, then tells the kernel the descriptors are free to reuse. consume_one_rx_descriptor() pops one entry from the used ring, advances our read cursor and the count of descriptors awaiting handoff, and returns a pointer to that descriptor's data. Bootstrap vhost in tap_sock_tun_init(). If setup fails we leave c->vhost.fd at -1 and carry on unaccelerated. With vhost-net running the kernel reads the tap fd itself, so don't add that fd to our epoll set at all: tap_handler_pasta() would otherwise be woken for data it must not read. Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- passt.c | 6 +++ tap.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- tap.h | 2 + 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/passt.c b/passt.c index 6002231..372b743 100644 --- a/passt.c +++ b/passt.c @@ -305,6 +305,12 @@ static void passt_worker(void *opaque, int nfds, struct epoll_event *events) case EPOLL_TYPE_CONF: conf_handler(c, eventmask); break; + case EPOLL_TYPE_VHOST_CALL: + tap_vhost_input(c, ref, &now); + break; + case EPOLL_TYPE_VHOST_ERROR: + die("Error on vhost-kernel socket"); + break; default: /* Can't happen */ assert(0); diff --git a/tap.c b/tap.c index dfa66c7..a5fdee8 100644 --- a/tap.c +++ b/tap.c @@ -38,6 +38,7 @@ #include <netinet/udp.h> #include <netinet/ip_icmp.h> #include <netinet/if_ether.h> +#include <sys/eventfd.h> #include <linux/if_tun.h> #include <linux/icmpv6.h> @@ -61,6 +62,7 @@ #include "vhost_user.h" #include "vu_common.h" #include "epoll_ctl.h" +#include "vhost.h" /* Maximum allowed frame lengths (including L2 header) */ @@ -1446,7 +1448,11 @@ static void tap_start_connection(const struct ctx *c) break; } - epoll_add(c->epollfd, EPOLLIN | EPOLLRDHUP, ref); + /* With vhost-net running, the kernel reads the tap fd for us and + * notifies us on the call eventfd, so we mustn't watch it ourselves + */ + if (c->vhost.fd == -1) + epoll_add(c->epollfd, EPOLLIN | EPOLLRDHUP, ref); if (!tap_is_ready(c)) return; @@ -1514,6 +1520,89 @@ void tap_listen_handler(struct ctx *c, uint32_t events) tap_start_connection(c); } +/** + * consume_one_rx_descriptor() - Consume one used from-guest descriptor + * @len: Set to the length of data written by the kernel + * + * Pops a single entry from the used ring, advancing our read cursor and the + * count of descriptors awaiting refill announcement. + * + * NOTE: This function assumes the kernel is going to post single descriptors + * always, No chains. If that changes, we would need to increment num_free + * as we advertise back to the kernel the free descriptors by the length of the chain. + * + * Return: pointer to the packet buffer, or NULL if no data is available + */ +static void *consume_one_rx_descriptor(size_t *len) +{ + struct vring_used_pasta *used = &vring_used_all[VHOST_AVAIL_Q_IDX]; + uint16_t used_idx, last_used; + uint32_t i; + + used_idx = le16toh(used->idx); + + smp_rmb(); + + /* If the kernel's used index matches ours, we've consumed everything + * it has posted + */ + if (used_idx == vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx) { + *len = 0; + return NULL; + } + + last_used = vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx % VHOST_NDESCS; + i = le32toh(used->ring[last_used].id); + *len = le32toh(used->ring[last_used].len); + + if (i != last_used) + die("vhost: id %u at used position %u", i, last_used); + + /* The kernel has queued for us something we cannot receive */ + if (*len > VHOST_DESC_BYTES) + die("vhost: id %u len %zu > %zu", i, *len, + (size_t)VHOST_DESC_BYTES); + + vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx++; + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free++; + + return pkt_buf + i * VHOST_DESC_BYTES; +} + +/** + * tap_vhost_input() - Handle frames the kernel wrote to the from-guest queue + * @c: Execution context + * @ref: epoll reference + * @now: Current timestamp + */ +void tap_vhost_input(struct ctx *c, union epoll_ref ref, + const struct timespec *now) +{ + struct virtio_net_hdr_mrg_rxbuf *hdr; + struct iov_tail data; + size_t len; + + eventfd_read(ref.fd, (eventfd_t[]){ 0 }); + + tap_flush_pools(); + + while ((hdr = consume_one_rx_descriptor(&len))) { + /* a valid receive is one where there is a virtio net header followed by atleast sizeof(struct ethhdr) bytes */ + if (len < (sizeof(*hdr) + sizeof(struct ethhdr))) { + warn("vhost: invalid len %lu", len); + continue; + } + + /* Skip over the vnet header, add the packet without it */ + data = IOV_TAIL_FROM_BUF((void *)(hdr + 1), + len - sizeof(*hdr), 0); + tap_add_packet(c, &data, now); + } + + tap_handler(c, now); + vhost_rx_descriptor_handoff(c); +} + /** * tap_ns_tun() - Get tuntap fd in namespace * @c: Execution context @@ -1555,12 +1644,33 @@ static int tap_ns_tun(void *arg) */ static void tap_sock_tun_init(struct ctx *c) { + int i; + if (!c->splice_only) { NS_CALL(tap_ns_tun, c); if (c->fd_tap == -1) die("Failed to set up tap device in namespace"); } + /* Nothing else here runs if this fails: c->vhost.fd stays -1 and we + * carry on reading the tap fd ourselves, unless acceleration was + * explicitly asked for + */ + if (c->vhost.mode != VHOST_MODE_OFF && vhost_setup_net(c) < 0 && + c->vhost.mode == VHOST_MODE_ON) + die("vhost-net is not available"); + + if (c->vhost.fd != -1) { + for (i = 0; i < ARRAY_SIZE(c->vhost.vq); i++) + vhost_setup_eventfds(c, i); + + if (vhost_setup_memory_table(c) < 0) + die_perror("VHOST_SET_MEM_TABLE ioctl failed"); + + for (i = 0; i < ARRAY_SIZE(c->vhost.vq); i++) + vhost_set_vring(c, i, c->fd_tap); + } + pasta_ns_conf(c); if (!c->splice_only) diff --git a/tap.h b/tap.h index f8c30e2..0f1f7a3 100644 --- a/tap.h +++ b/tap.h @@ -48,6 +48,8 @@ static inline void tap_hdr_update(struct tap_hdr *thdr, size_t l2len) thdr->vnet_len = htonl(l2len); } +void tap_vhost_input(struct ctx *c, union epoll_ref ref, + const struct timespec *now); unsigned long tap_l2_max_len(const struct ctx *c); void *tap_push_l2h(const struct ctx *c, void *buf, const void *src_mac, uint16_t proto); -- 2.39.5 (Apple Git-154) From da7ee374ec7d76dda971faffebb59a70cb9dff7c Mon Sep 17 00:00:00 2001 From: Ammar Yasser <aerosound161(a)gmail.com> Date: Fri, 4 Sep 2026 13:33:31 +0000 Subject: [PATCH v2 5/7] tap, tcp, udp: Prepare the to-guest path for vhost-net MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To: passt-dev(a)passt.top Cc: =?UTF-8?q?Eugenio=20P=C3=A9rez?= <eperezma(a)redhat.com> Frames sent through vhost-net carry a virtio-net header where pasta's tap header would otherwise go, this necessitates the following two changes: - udp_tap_prepare() has been modified to take a context parameter so it can check if vhost is set up. Only if it isn't should tap_hdr_update be called because this means that the active union variant is vnet_len no the virtio net header - tap_l2_offset() a new helper, has been introduced which tells pcap_multiple() how many bytes precede the L2 frame, which now varies with vhost as well as with the mode tap_send_single() calls tap_send_frames_passt() and tap_send_frames_pasta() directly rather than going through tap_send_frames(). Its callers - ARP, TCP resets, and DHCP, DHCPv6, NDP and ICMP via tap_udp*_send()/tap_icmp*_send() - build their frame in a buffer on the stack, not in the buffers registered with vhost-net. The kernel can only read from registered memory, so these protocols can never take the vhost path and shouldn't be tested for it on every frame. Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- tap.c | 112 ++++++++++++++++++++++++++++++++++-------------------- tcp_buf.c | 12 ++++-- udp.c | 27 +++++++++---- 3 files changed, 99 insertions(+), 52 deletions(-) diff --git a/tap.c b/tap.c index a5fdee8..fd500f1 100644 --- a/tap.c +++ b/tap.c @@ -123,45 +123,6 @@ unsigned long tap_l2_max_len(const struct ctx *c) return 0; /* Unreachable, for cppcheck's sake */ } -/** - * tap_send_single() - Send a single frame - * @c: Execution context - * @data: Packet buffer - * @l2len: Total L2 packet length - */ -void tap_send_single(const struct ctx *c, const void *data, size_t l2len) -{ - uint8_t padded[ETH_ZLEN] = { 0 }; - struct iovec iov[2]; - size_t iovcnt = 0; - uint32_t vnet_len; - - if (l2len < ETH_ZLEN) { - memcpy(padded, data, l2len); - data = padded; - l2len = ETH_ZLEN; - } - - vnet_len = htonl(l2len); - - switch (c->mode) { - case MODE_PASST: - iov[iovcnt] = IOV_OF_LVALUE(vnet_len); - iovcnt++; - /* fall through */ - case MODE_PASTA: - iov[iovcnt].iov_base = (void *)data; - iov[iovcnt].iov_len = l2len; - iovcnt++; - - tap_send_frames(c, iov, iovcnt, 1); - break; - case MODE_VU: - vu_send_single(c, data, l2len); - break; - } -} - /** * tap_push_l2h() - Build an L2 header for an inbound packet * @c: Execution context @@ -497,6 +458,76 @@ static size_t tap_send_frames_passt(const struct ctx *c, return i / bufs_per_frame; } +/** + * tap_l2_offset() - Backend specific header size preceding each L2 frame + * @c: Execution context + * + * Return: offset of the L2 frame within each frame's first buffer + */ +static size_t tap_l2_offset(const struct ctx *c) +{ + if (c->mode == MODE_PASST) + return sizeof(uint32_t); /* vnet_len */ + + if (c->vhost.fd != -1) + return VNET_HLEN; + + return 0; +} + +/** + * tap_send_single() - Send a single frame + * @c: Execution context + * @data: Packet buffer + * @l2len: Total L2 packet length + */ +void tap_send_single(const struct ctx *c, const void *data, size_t l2len) +{ + uint8_t padded[ETH_ZLEN] = { 0 }; + struct iovec iov[2]; + uint32_t vnet_len; + size_t iovcnt = 0; + size_t m = 0; + + if (l2len < ETH_ZLEN) { + memcpy(padded, data, l2len); + data = padded; + l2len = ETH_ZLEN; + } + + vnet_len = htonl(l2len); + + switch (c->mode) { + case MODE_PASST: + /* create an iov for the length */ + iov[iovcnt] = IOV_OF_LVALUE(vnet_len); + iovcnt++; + /* create the data iov */ + iov[iovcnt].iov_base = (void *)data; + iov[iovcnt].iov_len = l2len; + iovcnt++; + + m = tap_send_frames_passt(c, iov, iovcnt, 1); + break; + case MODE_PASTA: + /* don't create a length iov in the case of pasta */ + iov[iovcnt].iov_base = (void *)data; + iov[iovcnt].iov_len = l2len; + iovcnt++; + + m = tap_send_frames_pasta(c, iov, iovcnt, 1); + break; + case MODE_VU: + vu_send_single(c, data, l2len); + return; + } + + if (!m) + debug("tap: failed to send a single frame"); + + pcap_multiple(iov, iovcnt, m, tap_l2_offset(c)); +} + /** * tap_send_frames() - Send out multiple prepared frames * @c: Execution context @@ -537,8 +568,7 @@ size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, debug("tap: failed to send %zu frames of %zu", nframes - m, nframes); - pcap_multiple(iov, bufs_per_frame, m, - c->mode == MODE_PASST ? sizeof(uint32_t) : 0); + pcap_multiple(iov, bufs_per_frame, m, tap_l2_offset(c)); return m; } diff --git a/tcp_buf.c b/tcp_buf.c index dd80384..493c5c9 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -21,6 +21,7 @@ #include <netinet/ip.h> #include <netinet/tcp.h> +#include <linux/virtio_net.h> #include "util.h" #include "ip.h" @@ -44,7 +45,7 @@ /* Ethernet header for IPv4 and IPv6 frames */ static struct ethhdr tcp_eth_hdr[TCP_FRAMES_MEM]; -static struct tap_hdr tcp_payload_tap_hdr[TCP_FRAMES_MEM]; +static struct virtio_net_hdr_mrg_rxbuf tcp_payload_tap_hdr[TCP_FRAMES_MEM]; /* IP headers for IPv4 and IPv6 */ static struct iphdr tcp4_payload_ip[TCP_FRAMES_MEM]; @@ -113,7 +114,7 @@ void tcp_sock_iov_init(const struct ctx *c) for (i = 0; i < TCP_FRAMES_MEM; i++) { struct iovec *iov = tcp_l2_iov[i]; - iov[TCP_IOV_TAP] = tap_hdr_iov(c, &tcp_payload_tap_hdr[i]); + iov[TCP_IOV_TAP] = tap_hdr_iov(c, (struct tap_hdr *)&tcp_payload_tap_hdr[i]); iov[TCP_IOV_ETH].iov_len = sizeof(struct ethhdr); iov[TCP_IOV_PAYLOAD].iov_base = &tcp_payload[i]; iov[TCP_IOV_ETH_PAD].iov_base = eth_pad; @@ -216,7 +217,12 @@ static void tcp_l2_buf_fill_headers(const struct ctx *c, l2len = tcp_fill_headers(c, conn, eh, ip4h, ip6h, th, &tail, iov_tail_size(&tail), csum_flags, seq); - tap_hdr_update(taph, l2len); + + /* With vhost-net this buffer holds a virtio-net header, which a tap + * one must not be written over + */ + if (c->vhost.fd == -1) + tap_hdr_update(taph, l2len); } /** diff --git a/udp.c b/udp.c index a4355a1..afe1c7f 100644 --- a/udp.c +++ b/udp.c @@ -103,6 +103,7 @@ #include <time.h> #include <arpa/inet.h> #include <linux/errqueue.h> +#include <linux/virtio_net.h> #include "checksum.h" #include "util.h" @@ -149,12 +150,16 @@ static struct ethhdr udp_eth_hdr[UDP_MAX_FRAMES]; * struct udp_meta_t - Pre-cooked headers for UDP packets * @ip6h: Pre-filled IPv6 header (except for payload_len and addresses) * @ip4h: Pre-filled IPv4 header (except for tot_len and saddr) - * @taph: Tap backend specific header + * @vnet_hdr: virtio-net header, used when sending through vhost-net + * @taph: Tap backend specific header, used otherwise */ static struct udp_meta_t { struct ipv6hdr ip6h; struct iphdr ip4h; - struct tap_hdr taph; + union { + struct virtio_net_hdr_mrg_rxbuf vnet_hdr; + struct tap_hdr taph; + }; } #ifdef __AVX2__ __attribute__ ((aligned(32))) @@ -356,13 +361,15 @@ static void udp_tap_pad(struct iovec *iov) /** * udp_tap_prepare() - Convert one datagram into a tap frame + * @c: Execution context * @mmh: Receiving mmsghdr array * @idx: Index of the datagram to prepare * @tap_omac: MAC address of remote endpoint as seen from the guest * @toside: Flowside for destination side * @no_udp_csum: Do not set UDP checksum */ -static void udp_tap_prepare(const struct mmsghdr *mmh, +static void udp_tap_prepare(const struct ctx *c, + const struct mmsghdr *mmh, unsigned int idx, const uint8_t *tap_omac, const struct flowside *toside, @@ -384,8 +391,10 @@ static void udp_tap_prepare(const struct mmsghdr *mmh, udp_update_hdr6(&bm->ip6h, uh, &payload, toside, mmh[idx].msg_len, no_udp_csum); - l2len = MAX(l4len + sizeof(bm->ip6h) + ETH_HLEN, ETH_ZLEN); - tap_hdr_update(&bm->taph, l2len); + if (c->mode == MODE_PASST) { + l2len = MAX(l4len + sizeof(bm->ip6h) + ETH_HLEN, ETH_ZLEN); + tap_hdr_update(&bm->taph, l2len); + } eh->h_proto = htons_constant(ETH_P_IPV6); (*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip6h); @@ -393,8 +402,10 @@ static void udp_tap_prepare(const struct mmsghdr *mmh, udp_update_hdr4(&bm->ip4h, uh, &payload, toside, mmh[idx].msg_len, no_udp_csum); - l2len = MAX(l4len + sizeof(bm->ip4h) + ETH_HLEN, ETH_ZLEN); - tap_hdr_update(&bm->taph, l2len); + if (c->mode == MODE_PASST) { + l2len = MAX(l4len + sizeof(bm->ip4h) + ETH_HLEN, ETH_ZLEN); + tap_hdr_update(&bm->taph, l2len); + } eh->h_proto = htons_constant(ETH_P_IP); (*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip4h); @@ -867,7 +878,7 @@ static void udp_buf_sock_to_tap(const struct ctx *c, int s, int n, fwd_neigh_mac_get(c, &toside->oaddr, omac); for (i = 0; i < n; i++) - udp_tap_prepare(udp_mh_recv, i, omac, toside, false); + udp_tap_prepare(c, udp_mh_recv, i, omac, toside, false); tap_send_frames(c, &udp_l2_iov[0][0], UDP_NUM_IOVS, n); } -- 2.39.5 (Apple Git-154) From d616724386df38c474d304508da64d1be416b5cd Mon Sep 17 00:00:00 2001 From: Ammar Yasser <aerosound161(a)gmail.com> Date: Fri, 4 Sep 2026 13:47:09 +0000 Subject: [PATCH v2 6/7] tap: Implement the pasta vhost-net to-guest path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To: passt-dev(a)passt.top Cc: =?UTF-8?q?Eugenio=20P=C3=A9rez?= <eperezma(a)redhat.com> Add a vhost argument to tap_send_frames_pasta(). Callers pass it to say whether this send should go through vhost-net, and when it does sending goes through tap_send_frames_vhost(), which takes a descriptor from the queue shared with the kernel for each iov and points it at that iov's base address, chaining the bufs_per_frame descriptors of a frame together so the guest receives it in one go. tx_reap() reclaims the descriptors the kernel has finished with, walking each used chain to its end so that every descriptor in it is counted as free again. Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- tap.c | 150 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 147 insertions(+), 3 deletions(-) diff --git a/tap.c b/tap.c index fd500f1..bc2700f 100644 --- a/tap.c +++ b/tap.c @@ -359,12 +359,151 @@ void tap_icmp6_send(const struct ctx *c, tap_send_single(c, buf, l4len + ((char *)icmp6h - buf)); } +/** + * tx_reap() - Reclaim the descriptors the kernel has already processed + */ +static void tx_reap(void) +{ + struct vring_used_pasta *used = &vring_used_all[VHOST_USED_Q_IDX]; + uint16_t used_idx = le16toh(used->idx); + + smp_rmb(); + + /* increment last_used_idx until it reaches the kernel's used index */ + while (vhost_vq_state[VHOST_USED_Q_IDX].last_used_idx != used_idx) { + uint16_t last_used; + uint32_t desc_id; + + last_used = vhost_vq_state[VHOST_USED_Q_IDX].last_used_idx % VHOST_NDESCS; + desc_id = le32toh(used->ring[last_used].id); + + for (;;) { + /* keep going until we find a descriptor without the + * next flag + */ + vhost_vq_state[VHOST_USED_Q_IDX].num_free++; + if (!(le16toh(vring_desc[VHOST_USED_Q_IDX][desc_id].flags) & + VRING_DESC_F_NEXT)) + break; + + /* this descriptor wasn't the last, set desc_id to the + * next one and keep going + */ + desc_id = le32toh(vring_desc[VHOST_USED_Q_IDX][desc_id].next); + } + + vhost_vq_state[VHOST_USED_Q_IDX].last_used_idx++; + } +} + +/** + * tap_send_frames_vhost() - Send multiple frames to the pasta tap + * @c: Execution context + * @iov: Array of buffers + * @bufs_per_frame: Number of buffers (iovec entries) per frame + * @nframes: Number of frames to send + * + * @iov must have total length @bufs_per_frame * @nframes, with each set of + * @bufs_per_frame contiguous buffers representing a single frame. + * + * Return: number of frames successfully sent + */ +static size_t tap_send_frames_vhost(const struct ctx *c, + const struct iovec *iov, + size_t bufs_per_frame, size_t nframes) +{ + struct vring_avail_pasta *avail = + &vring_avail_all[VHOST_USED_Q_IDX]; + size_t processed_frames = 0; + size_t i; + + /* reclaim descriptors if we don't have enough available buffers to + * perform this send + */ + if (vhost_vq_state[VHOST_USED_Q_IDX].num_free < bufs_per_frame * nframes) + tx_reap(); + + for (i = 0; i < nframes; i++) { + uint16_t head; + size_t j; + + /* it's likely that tx_reap returned to us less than + * bufs_per_frame descs + */ + if (vhost_vq_state[VHOST_USED_Q_IDX].num_free < bufs_per_frame) + break; + + /* set the index of the avail ring in the tx queue to be our + * last_used_idx + */ + head = vhost_vq_state[VHOST_USED_Q_IDX].next_free % VHOST_NDESCS; + avail->ring[(avail->idx + i) % VHOST_NDESCS] = htole16(head); + + /* we will be consuming bufs_per_frame descriptors for every + * frame, decrement the local num_free + */ + vhost_vq_state[VHOST_USED_Q_IDX].num_free -= bufs_per_frame; + + for (j = 0; j < bufs_per_frame; ++j) { + uint16_t next = + vhost_vq_state[VHOST_USED_Q_IDX].next_free % + VHOST_NDESCS; + /* get the last_used_idx descriptor */ + struct vring_desc_pasta *desc = + &vring_desc[VHOST_USED_Q_IDX][next]; + const struct iovec *iov_i; + + /* the iov variable contains the iovecs for all frames + * we will send. access a single fragment of a frame + * (each fragment is one of tcp_iov_parts) denoted by + * iov at index i (index of the frame being processed) + * * bufs_per_frame plus j (the index of the fragment + * being processed) + */ + iov_i = &iov[i * bufs_per_frame + j]; + + /* set that descriptor's address to the base of the iov + * and set the VRING_DESC_F_NEXT flag on the descriptor + * if its not the last frame fragment, so that the + * guest would recieve the entire frame in one go. + */ + desc->addr = (uint64_t)iov_i->iov_base; + desc->len = iov_i->iov_len; + desc->flags = (j == bufs_per_frame - 1) ? + 0 : htole16(VRING_DESC_F_NEXT); + + vhost_vq_state[VHOST_USED_Q_IDX].next_free++; + } + + processed_frames++; + } + + /* we didn't process any frames, no need to notify the kernel */ + if (!processed_frames) + return 0; + + smp_wmb(); + + /* we will have used nframes descriptor chains */ + avail->idx = htole16(le16toh(avail->idx) + processed_frames); + + vhost_kick(&vring_used_all[VHOST_USED_Q_IDX], + c->vhost.vq[VHOST_USED_Q_IDX].kick_fd); + + /* wait until the kernel finishes processing this send */ + while (avail->idx != vring_used_all[1].used.idx) + ; + + return processed_frames; +} + /** * tap_send_frames_pasta() - Send multiple frames to the pasta tap * @c: Execution context * @iov: Array of buffers * @bufs_per_frame: Number of buffers (iovec entries) per frame * @nframes: Number of frames to send + * @vhost: Send through vhost-net rather than writing to the tap * * @iov must have total length @bufs_per_frame * @nframes, with each set of * @bufs_per_frame contiguous buffers representing a single frame. @@ -375,11 +514,15 @@ void tap_icmp6_send(const struct ctx *c, */ static size_t tap_send_frames_pasta(const struct ctx *c, const struct iovec *iov, - size_t bufs_per_frame, size_t nframes) + size_t bufs_per_frame, size_t nframes, + bool vhost) { size_t nbufs = bufs_per_frame * nframes; size_t i; + if (vhost) + return tap_send_frames_vhost(c, iov, bufs_per_frame, nframes); + for (i = 0; i < nbufs; i += bufs_per_frame) { ssize_t rc = writev(c->fd_tap, iov + i, bufs_per_frame); size_t framelen = iov_size(iov + i, bufs_per_frame); @@ -515,7 +658,7 @@ void tap_send_single(const struct ctx *c, const void *data, size_t l2len) iov[iovcnt].iov_len = l2len; iovcnt++; - m = tap_send_frames_pasta(c, iov, iovcnt, 1); + m = tap_send_frames_pasta(c, iov, iovcnt, 1, false); break; case MODE_VU: vu_send_single(c, data, l2len); @@ -553,7 +696,8 @@ size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, switch (c->mode) { case MODE_PASTA: - m = tap_send_frames_pasta(c, iov, bufs_per_frame, nframes); + m = tap_send_frames_pasta(c, iov, bufs_per_frame, nframes, + c->vhost.fd != -1); break; case MODE_PASST: m = tap_send_frames_passt(c, iov, bufs_per_frame, nframes); -- 2.39.5 (Apple Git-154) From 5be079ef9866a26be9c57f1c06597ae61d0a5d36 Mon Sep 17 00:00:00 2001 From: Ammar Yasser <aerosound161(a)gmail.com> Date: Fri, 4 Sep 2026 14:02:27 +0000 Subject: [PATCH v2 7/7] tap/tcp: Replace tcp_payload_used with a ring buffer style index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To: passt-dev(a)passt.top Cc: =?UTF-8?q?Eugenio=20P=C3=A9rez?= <eperezma(a)redhat.com> Because pasta resets tcp_payload_used to 0 after every tcp flush this means that pasta will very quickly override data in the tcp buffers after sending. This is not a problem when using writev because the data will have been copied elsewhere by the kernel, but this is a problem for vhost where the kernel thread may not have finished processing this data. This is why a synchronous send was required in vhost, which obviously affects performance. Replace tcp_payload_used with a struct tcp_payload_idx holding the last index we wrote to and a counter that gets updated when we flush packets, from where we last flushed up to that index. Both are grow only and access gets modded by TCP_FRAMES_MEM. Rewrite functions in tcp_buf.c to use that index and handle the wraparound case (sending a batch that started near the end of the buffers and continued to be at the start) in tcp_payload_flush(). With the buffers no longer reused from under the kernel, drop the synchronous wait from tap_send_frames_vhost() and reclaim descriptors until there are enough for the send instead. Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- tap.c | 6 +-- tcp_buf.c | 115 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/tap.c b/tap.c index bc2700f..0456589 100644 --- a/tap.c +++ b/tap.c @@ -490,10 +490,6 @@ static size_t tap_send_frames_vhost(const struct ctx *c, vhost_kick(&vring_used_all[VHOST_USED_Q_IDX], c->vhost.vq[VHOST_USED_Q_IDX].kick_fd); - /* wait until the kernel finishes processing this send */ - while (avail->idx != vring_used_all[1].used.idx) - ; - return processed_frames; } @@ -521,7 +517,7 @@ static size_t tap_send_frames_pasta(const struct ctx *c, size_t i; if (vhost) - return tap_send_frames_vhost(c, iov, bufs_per_frame, nframes); + return tap_send_frames_vhost(c, iov, bufs_per_frame, nframes); for (i = 0; i < nbufs; i += bufs_per_frame) { ssize_t rc = writev(c->fd_tap, iov + i, bufs_per_frame); diff --git a/tcp_buf.c b/tcp_buf.c index 493c5c9..425410f 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -59,7 +59,22 @@ static_assert(MSS6 <= sizeof(tcp_payload[0].data), "MSS6 is greater than 65516") /* References tracking the owner connection of frames in the tap outqueue */ static struct tcp_tap_conn *tcp_frame_conns[TCP_FRAMES_MEM]; -static unsigned int tcp_payload_used; + +/** + * struct tcp_payload_idx - Grow-only cursors into the TCP frame buffers + * @tcp_buf_idx: The last index we wrote data to in the tcp buffers + * @processed: Last index we pushed to the underlying connection + */ +static struct tcp_payload_idx { + unsigned int tcp_buf_idx; + unsigned int processed; +} tcp_idx; + +/* latest index we wrote to into the tcp buffers */ +#define TCP_CURR_IDX (tcp_idx.tcp_buf_idx % TCP_FRAMES_MEM) + +/* current amount of frames queued in tcp buffers but not sent */ +#define TCP_FRAME_COUNT (tcp_idx.tcp_buf_idx - tcp_idx.processed) /* recvmsg()/sendmsg() data for tap */ static struct iovec iov_sock [TCP_FRAMES_MEM + DISCARD_IOV_NUM]; @@ -158,15 +173,46 @@ static void tcp_revert_seq(const struct ctx *c, struct tcp_tap_conn **conns, */ void tcp_payload_flush(const struct ctx *c, const struct timespec *now) { - size_t m; + unsigned int total = TCP_FRAME_COUNT; + unsigned int start = tcp_idx.processed % TCP_FRAMES_MEM; + unsigned int first_batch_size, sent; - m = tap_send_frames(c, &tcp_l2_iov[0][0], TCP_NUM_IOVS, - tcp_payload_used); - if (m != tcp_payload_used) { - tcp_revert_seq(c, &tcp_frame_conns[m], &tcp_l2_iov[m], - tcp_payload_used - m, now); + if (!total) + return; + + /* What is smaller ? all we what we want to send ? or 128 - the index + * indicating the start of where we wrote this batch ? + */ + first_batch_size = MIN(total, TCP_FRAMES_MEM - start); + + sent = tap_send_frames(c, &tcp_l2_iov[start][0], TCP_NUM_IOVS, + first_batch_size); + if (sent < first_batch_size) { + tcp_revert_seq(c, &tcp_frame_conns[start + sent], + &tcp_l2_iov[start + sent], + first_batch_size - sent, now); + goto out; } - tcp_payload_used = 0; + + /* There was more data to send than from tcp_idx.processed up to 128. + * the rest of the batch is going to be at index 0 up total - + * first_batch_size. + */ + if (total > first_batch_size) { + unsigned int second_batch_size = total - first_batch_size; + size_t m2; + + m2 = tap_send_frames(c, &tcp_l2_iov[0][0], TCP_NUM_IOVS, + second_batch_size); + sent += m2; + + if (m2 < second_batch_size) + tcp_revert_seq(c, &tcp_frame_conns[m2], &tcp_l2_iov[m2], + second_batch_size - m2, now); + } + +out: + tcp_idx.processed += sent; } /** @@ -244,13 +290,13 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, uint32_t seq; int ret; - iov = tcp_l2_iov[tcp_payload_used]; + iov = tcp_l2_iov[TCP_CURR_IDX]; if (CONN_V4(conn)) - iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[tcp_payload_used]); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[TCP_CURR_IDX]); else - iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[TCP_CURR_IDX]); - iov[TCP_IOV_ETH] = IOV_OF_LVALUE(tcp_eth_hdr[tcp_payload_used]); + iov[TCP_IOV_ETH] = IOV_OF_LVALUE(tcp_eth_hdr[TCP_CURR_IDX]); payload = iov[TCP_IOV_PAYLOAD].iov_base; seq = conn->seq_to_tap; ret = tcp_prepare_flags(c, conn, flags, &payload->th, @@ -259,7 +305,8 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, if (ret <= 0) return ret; - tcp_frame_conns[tcp_payload_used++] = conn; + tcp_idx.tcp_buf_idx++; + tcp_frame_conns[TCP_CURR_IDX] = conn; l4len = optlen + sizeof(struct tcphdr); iov[TCP_IOV_PAYLOAD].iov_len = l4len; @@ -271,8 +318,9 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, tcp_l2_buf_pad(iov); if (flags & DUP_ACK) { - struct iovec *dup_iov = tcp_l2_iov[tcp_payload_used]; - tcp_frame_conns[tcp_payload_used++] = conn; + struct iovec *dup_iov = tcp_l2_iov[TCP_CURR_IDX]; + tcp_frame_conns[TCP_CURR_IDX] = conn; + tcp_idx.tcp_buf_idx++; memcpy(dup_iov[TCP_IOV_TAP].iov_base, iov[TCP_IOV_TAP].iov_base, iov[TCP_IOV_TAP].iov_len); @@ -284,7 +332,7 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, dup_iov[TCP_IOV_ETH_PAD].iov_len = iov[TCP_IOV_ETH_PAD].iov_len; } - if (tcp_payload_used > TCP_FRAMES_MEM - 2) + if (TCP_FRAME_COUNT > TCP_FRAMES_MEM - 2) tcp_payload_flush(c, now); return 0; @@ -309,21 +357,29 @@ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn, struct iovec *iov; conn->seq_to_tap = seq + dlen; - tcp_frame_conns[tcp_payload_used] = conn; - iov = tcp_l2_iov[tcp_payload_used]; + tcp_frame_conns[TCP_CURR_IDX] = conn; + iov = tcp_l2_iov[TCP_CURR_IDX]; if (CONN_V4(conn)) { if (no_csum) { - struct iovec *iov_prev = tcp_l2_iov[tcp_payload_used - 1]; + /* TCP_CURR_IDX may be zero if the underlying + * tcp_idx.tcp_buf_idx is a multiple of 128, minus one + * will yield an invalid index. The previous index to 0 + * is 127. + */ + unsigned int prev_idx = (TCP_CURR_IDX + + TCP_FRAMES_MEM - 1) % + TCP_FRAMES_MEM; + struct iovec *iov_prev = tcp_l2_iov[prev_idx]; const struct iphdr *iph = iov_prev[TCP_IOV_IP].iov_base; /* overwrite IP4_CSUM flag as we set the checksum */ check = iph->check; } - iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[tcp_payload_used]); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[TCP_CURR_IDX]); } else if (CONN_V6(conn)) { - iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[TCP_CURR_IDX]); } - iov[TCP_IOV_ETH].iov_base = &tcp_eth_hdr[tcp_payload_used]; + iov[TCP_IOV_ETH].iov_base = &tcp_eth_hdr[TCP_CURR_IDX]; payload = iov[TCP_IOV_PAYLOAD].iov_base; payload->th.th_off = sizeof(struct tcphdr) / 4; payload->th.th_x2 = 0; @@ -335,7 +391,8 @@ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn, tcp_l2_buf_pad(iov); - if (++tcp_payload_used > TCP_FRAMES_MEM - 1) + tcp_idx.tcp_buf_idx++; + if (TCP_FRAME_COUNT > TCP_FRAMES_MEM - 1) tcp_payload_flush(c, now); } @@ -375,15 +432,13 @@ int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, return -1; } - if (tcp_payload_used + fill_bufs > TCP_FRAMES_MEM) { + if (TCP_FRAME_COUNT + (unsigned int)fill_bufs > TCP_FRAMES_MEM) tcp_payload_flush(c, now); - /* Silence Coverity CWE-125 false positive */ - tcp_payload_used = 0; - } - for (i = 0, iov = iov_sock + DISCARD_IOV_NUM; i < fill_bufs; i++, iov++) { - iov->iov_base = &tcp_payload[tcp_payload_used + i].data; + unsigned int idx = (TCP_CURR_IDX + i) % TCP_FRAMES_MEM; + + iov->iov_base = &tcp_payload[idx].data; iov->iov_len = mss; } if (iov_rem) @@ -457,7 +512,7 @@ int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, dlen = mss; seq = conn->seq_to_tap; for (i = 0; i < send_bufs; i++) { - int no_csum = i && i != send_bufs - 1 && tcp_payload_used; + int no_csum = i && i != send_bufs - 1; bool push = false; if (i == send_bufs - 1) { -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 4/7] tap: Implement the pasta vhost-net from-guest path
by Ammar Yasser 16 Oct '26

16 Oct '26

Add tap_vhost_input(), called when the kernel signals on a call eventfd that it has written guest frames into the from-guest queue. It consumes descriptors with consume_one_rx_descriptor(), skips the virtio_net header on each, builds an iov_tail from what's left and queues it for processing, then tells the kernel the descriptors are free to reuse. consume_one_rx_descriptor() pops one entry from the used ring, advances our read cursor and the count of descriptors awaiting handoff, and returns a pointer to that descriptor's data. Bootstrap vhost in tap_sock_tun_init(). If setup fails we leave c->vhost.fd at -1 and carry on unaccelerated. With vhost-net running the kernel reads the tap fd itself, so don't add that fd to our epoll set at all: tap_handler_pasta() would otherwise be woken for data it must not read. Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- passt.c | 6 +++ tap.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- tap.h | 2 + 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/passt.c b/passt.c index 6002231..372b743 100644 --- a/passt.c +++ b/passt.c @@ -305,6 +305,12 @@ static void passt_worker(void *opaque, int nfds, struct epoll_event *events) case EPOLL_TYPE_CONF: conf_handler(c, eventmask); break; + case EPOLL_TYPE_VHOST_CALL: + tap_vhost_input(c, ref, &now); + break; + case EPOLL_TYPE_VHOST_ERROR: + die("Error on vhost-kernel socket"); + break; default: /* Can't happen */ assert(0); diff --git a/tap.c b/tap.c index dfa66c7..a5fdee8 100644 --- a/tap.c +++ b/tap.c @@ -38,6 +38,7 @@ #include <netinet/udp.h> #include <netinet/ip_icmp.h> #include <netinet/if_ether.h> +#include <sys/eventfd.h> #include <linux/if_tun.h> #include <linux/icmpv6.h> @@ -61,6 +62,7 @@ #include "vhost_user.h" #include "vu_common.h" #include "epoll_ctl.h" +#include "vhost.h" /* Maximum allowed frame lengths (including L2 header) */ @@ -1446,7 +1448,11 @@ static void tap_start_connection(const struct ctx *c) break; } - epoll_add(c->epollfd, EPOLLIN | EPOLLRDHUP, ref); + /* With vhost-net running, the kernel reads the tap fd for us and + * notifies us on the call eventfd, so we mustn't watch it ourselves + */ + if (c->vhost.fd == -1) + epoll_add(c->epollfd, EPOLLIN | EPOLLRDHUP, ref); if (!tap_is_ready(c)) return; @@ -1514,6 +1520,89 @@ void tap_listen_handler(struct ctx *c, uint32_t events) tap_start_connection(c); } +/** + * consume_one_rx_descriptor() - Consume one used from-guest descriptor + * @len: Set to the length of data written by the kernel + * + * Pops a single entry from the used ring, advancing our read cursor and the + * count of descriptors awaiting refill announcement. + * + * NOTE: This function assumes the kernel is going to post single descriptors + * always, No chains. If that changes, we would need to increment num_free + * as we advertise back to the kernel the free descriptors by the length of the chain. + * + * Return: pointer to the packet buffer, or NULL if no data is available + */ +static void *consume_one_rx_descriptor(size_t *len) +{ + struct vring_used_pasta *used = &vring_used_all[VHOST_AVAIL_Q_IDX]; + uint16_t used_idx, last_used; + uint32_t i; + + used_idx = le16toh(used->idx); + + smp_rmb(); + + /* If the kernel's used index matches ours, we've consumed everything + * it has posted + */ + if (used_idx == vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx) { + *len = 0; + return NULL; + } + + last_used = vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx % VHOST_NDESCS; + i = le32toh(used->ring[last_used].id); + *len = le32toh(used->ring[last_used].len); + + if (i != last_used) + die("vhost: id %u at used position %u", i, last_used); + + /* The kernel has queued for us something we cannot receive */ + if (*len > VHOST_DESC_BYTES) + die("vhost: id %u len %zu > %zu", i, *len, + (size_t)VHOST_DESC_BYTES); + + vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx++; + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free++; + + return pkt_buf + i * VHOST_DESC_BYTES; +} + +/** + * tap_vhost_input() - Handle frames the kernel wrote to the from-guest queue + * @c: Execution context + * @ref: epoll reference + * @now: Current timestamp + */ +void tap_vhost_input(struct ctx *c, union epoll_ref ref, + const struct timespec *now) +{ + struct virtio_net_hdr_mrg_rxbuf *hdr; + struct iov_tail data; + size_t len; + + eventfd_read(ref.fd, (eventfd_t[]){ 0 }); + + tap_flush_pools(); + + while ((hdr = consume_one_rx_descriptor(&len))) { + /* a valid receive is one where there is a virtio net header followed by atleast sizeof(struct ethhdr) bytes */ + if (len < (sizeof(*hdr) + sizeof(struct ethhdr))) { + warn("vhost: invalid len %lu", len); + continue; + } + + /* Skip over the vnet header, add the packet without it */ + data = IOV_TAIL_FROM_BUF((void *)(hdr + 1), + len - sizeof(*hdr), 0); + tap_add_packet(c, &data, now); + } + + tap_handler(c, now); + vhost_rx_descriptor_handoff(c); +} + /** * tap_ns_tun() - Get tuntap fd in namespace * @c: Execution context @@ -1555,12 +1644,33 @@ static int tap_ns_tun(void *arg) */ static void tap_sock_tun_init(struct ctx *c) { + int i; + if (!c->splice_only) { NS_CALL(tap_ns_tun, c); if (c->fd_tap == -1) die("Failed to set up tap device in namespace"); } + /* Nothing else here runs if this fails: c->vhost.fd stays -1 and we + * carry on reading the tap fd ourselves, unless acceleration was + * explicitly asked for + */ + if (c->vhost.mode != VHOST_MODE_OFF && vhost_setup_net(c) < 0 && + c->vhost.mode == VHOST_MODE_ON) + die("vhost-net is not available"); + + if (c->vhost.fd != -1) { + for (i = 0; i < ARRAY_SIZE(c->vhost.vq); i++) + vhost_setup_eventfds(c, i); + + if (vhost_setup_memory_table(c) < 0) + die_perror("VHOST_SET_MEM_TABLE ioctl failed"); + + for (i = 0; i < ARRAY_SIZE(c->vhost.vq); i++) + vhost_set_vring(c, i, c->fd_tap); + } + pasta_ns_conf(c); if (!c->splice_only) diff --git a/tap.h b/tap.h index f8c30e2..0f1f7a3 100644 --- a/tap.h +++ b/tap.h @@ -48,6 +48,8 @@ static inline void tap_hdr_update(struct tap_hdr *thdr, size_t l2len) thdr->vnet_len = htonl(l2len); } +void tap_vhost_input(struct ctx *c, union epoll_ref ref, + const struct timespec *now); unsigned long tap_l2_max_len(const struct ctx *c); void *tap_push_l2h(const struct ctx *c, void *buf, const void *src_mac, uint16_t proto); -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 3/7] virtio: Add the pasta vhost-net interface and implementation
by Ammar Yasser 16 Oct '26

16 Oct '26

Add vhost.c and vhost.h which define the api and implementation for pasta vhost acceleration - vq_state/vhost_vq_state[]: our own view of each virtqueue, which the kernel doesn't track for us: descriptors ready to hand back, how far we've read the used ring, and the next descriptor to allocate - vring_desc, vring_avail_all, vring_used_all, vhost_memory: the rings and the region table shared with the kernel - vhost_setup_net: open /dev/vhost-net and negotiate VIRTIO_F_VERSION_1 and VHOST_NET_F_VIRTIO_NET_HDR. Failure is fatal only if --vhost-kernel is set to 'on'. - vhost_setup_eventfds: create, register and watch the call, kick and error eventfds for one queue - vhost_setup_memory_table: register the buffers the kernel may touch. pasta has no guest, so each region maps guest_phys_addr onto the identical userspace_addr, making GPA translation an identity mapping. This function uses the *_register_memory_regions functions created per protocol and for general purpose buffers. - vhost_set_vring: publish a vring's addresses, bind the tap fd as its backend and lay out its descriptors - vhost_rx_descriptor_handoff, vhost_kick: hand consumed descriptors back and notify the kernel unless it says it's already polling Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- Makefile | 2 +- tcp_buf.c | 22 +++- tcp_buf.h | 4 + udp.c | 20 ++++ udp.h | 3 + util.c | 18 ++++ util.h | 2 + vhost.c | 295 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ vhost.h | 91 +++++++++++++++++ 9 files changed, 455 insertions(+), 2 deletions(-) create mode 100644 vhost.c create mode 100644 vhost.h diff --git a/Makefile b/Makefile index f6c87b1..6f11e4b 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ PASST_SRCS = arch.c arp.c bitmap.c checksum.c conf.c dhcp.c dhcpv6.c \ isolation.c lineread.c log.c mld.c ndp.c netlink.c migrate.c packet.c \ parse.c passt.c pasta.c pcap.c pif.c repair.c serialise.c tap.c tcp.c \ tcp_buf.c tcp_splice.c tcp_vu.c udp.c udp_flow.c udp_vu.c util.c \ - vhost_user.c virtio.c vu_common.c + vhost.c vhost_user.c virtio.c vu_common.c PASST_REPAIR_SRCS = passt-repair.c PESTO_SRCS = pesto.c bitmap.c fwd_rule.c inany.c ip.c lineread.c parse.c \ serialise.c diff --git a/tcp_buf.c b/tcp_buf.c index 72c4541..dd80384 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -31,12 +31,14 @@ #include "inany.h" #include "tcp_conn.h" #include "tcp_internal.h" -#include "tcp_buf.h" +#include "vhost.h" #define TCP_FRAMES_MEM 128 #define TCP_FRAMES \ (c->mode == MODE_PASTA ? 1 : TCP_FRAMES_MEM) +#define TCP_NREGIONS 5 + /* Static buffers */ /* Ethernet header for IPv4 and IPv6 frames */ @@ -75,6 +77,24 @@ void tcp_update_l2_buf(const unsigned char *eth_d) eth_update_mac(&tcp_eth_hdr[i], eth_d, NULL); } +/** + * tcp_register_memory_region() - Register the TCP specific buffers into a memory + * struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void tcp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + TCP_NREGIONS) > N_VHOST_REGIONS) + die("tcp: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + TCP_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_payload_tap_hdr); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp4_payload_ip); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp6_payload_ip); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_payload); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_eth_hdr); +} + /** * tcp_sock_iov_init() - Initialise scatter-gather L2 buffers for IPv4 sockets * @c: Execution context diff --git a/tcp_buf.h b/tcp_buf.h index 5d31cea..5e341ed 100644 --- a/tcp_buf.h +++ b/tcp_buf.h @@ -6,11 +6,15 @@ #ifndef TCP_BUF_H #define TCP_BUF_H +union vhost_memory_u; +struct tcp_tap_conn; + void tcp_sock_iov_init(const struct ctx *c); void tcp_payload_flush(const struct ctx *c, const struct timespec *now); int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, uint32_t already_sent, const struct timespec *now); int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, const struct timespec *now); +void tcp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); #endif /*TCP_BUF_H */ diff --git a/udp.c b/udp.c index 505e554..a4355a1 100644 --- a/udp.c +++ b/udp.c @@ -118,6 +118,7 @@ #include "udp_internal.h" #include "udp_vu.h" #include "epoll_ctl.h" +#include "vhost.h" #define UDP_MAX_FRAMES 32 /* max # of frames to receive at once */ @@ -128,6 +129,8 @@ #define UDP_TIMEOUT_DEFAULT 30 /* s */ #define UDP_TIMEOUT_STREAM_DEFAULT 120 /* s */ +#define UDP_NREGIONS 5 + /* Maximum UDP data to be returned in ICMP messages */ #define ICMP4_MAX_DLEN 8 #define ICMP6_MAX_DLEN (IPV6_MIN_MTU \ @@ -448,6 +451,23 @@ static void udp_send_tap_icmp4(const struct ctx *c, tap_icmp4_send(c, saddr, eaddr, &msg, tap_omac, msglen); } +/** + * udp_register_memory_region() - Register the UDP specific buffers into a memory + * struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void udp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + UDP_NREGIONS) > N_VHOST_REGIONS) + die("udp: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + UDP_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_payload); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_eth_hdr); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_iov_recv); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_mh_recv); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_meta); +} /** * udp_send_tap_icmp6() - Construct and send ICMPv6 to local peer diff --git a/udp.h b/udp.h index b50283e..ebf1a4e 100644 --- a/udp.h +++ b/udp.h @@ -11,6 +11,8 @@ #include "fwd.h" +union vhost_memory_u; + void udp_listen_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, const struct timespec *now); void udp_sock_handler(const struct ctx *c, union epoll_ref ref, @@ -21,6 +23,7 @@ int udp_tap_handler(const struct ctx *c, uint8_t pif, const struct timespec *now); int udp_init(struct ctx *c); void udp_update_l2_buf(const unsigned char *eth_d); +void udp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); /** * struct udp_ctx - Execution context for UDP diff --git a/util.c b/util.c index ff1f19f..cafe0ec 100644 --- a/util.c +++ b/util.c @@ -33,10 +33,13 @@ #include "log.h" #include "pcap.h" #include "serialise.h" +#include "vhost.h" #ifdef HAS_GETRANDOM #include <sys/random.h> #endif +#define GENERAL_NREGIONS 2 + /* Zero-filled buffer to pad 802.3 frames, up to 60 (ETH_ZLEN) bytes */ uint8_t eth_pad[ETH_ZLEN] = { 0 }; @@ -453,6 +456,21 @@ int open_in_ns(const struct ctx *c, const char *path, int flags) return arg.fd; } +/** + * general_register_memory_region() - Register the general (non protocol specific) buffers + * into a memory struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void general_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + GENERAL_NREGIONS) > N_VHOST_REGIONS) + die("general: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + GENERAL_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(pkt_buf); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(eth_pad); +} + /** * pidfile_write() - Write PID to file, if requested to do so, and close it * @fd: Open PID file descriptor, closed on exit, -1 to skip writing it diff --git a/util.h b/util.h index ec2d9ac..11f6877 100644 --- a/util.h +++ b/util.h @@ -145,6 +145,7 @@ int do_clone(int (*fn)(void *), char *stack_area, size_t stack_size, int flags, struct ctx; union sockaddr_inany; +union vhost_memory_u; int sock_l4(const struct ctx *c, enum epoll_type type, const union sockaddr_inany *sa, const char *ifname); @@ -170,6 +171,7 @@ int write_remainder(int fd, const struct iovec *iov, size_t iovcnt, int read_remainder(int fd, const struct iovec *iov, size_t cnt, size_t skip); bool snprintf_check(char *str, size_t size, const char *format, ...); long clamped_scale(long x, long y, long lo, long hi, long f); +void general_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); /** * af_name() - Return name of an address family diff --git a/vhost.c b/vhost.c new file mode 100644 index 0000000..a69c4b5 --- /dev/null +++ b/vhost.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Copyright Red Hat + * Author: Ammar Yasser <aerosound161(a)gmail.com> + * + * vhost.c - vhost-net (vhost-kernel) acceleration for pasta mode + */ + +#include <errno.h> +#include <fcntl.h> +#include <inttypes.h> +#include <stdint.h> +#include <string.h> +#include <unistd.h> +#include <sys/eventfd.h> +#include <sys/ioctl.h> + +#include "util.h" +#include "passt.h" +#include "vhost.h" +#include "epoll_ctl.h" +#include "udp.h" +#include "tcp_buf.h" + +struct vq_state vhost_vq_state[2]; + +struct vring_desc_pasta vring_desc[2][VHOST_NDESCS] + __attribute__((aligned(PAGE_SIZE))); + +struct vring_avail_pasta vring_avail_all[2] + __attribute__((aligned(PAGE_SIZE))); +struct vring_used_pasta vring_used_all[2] + __attribute__((aligned(PAGE_SIZE))); + +union vhost_memory_u vhost_memory = { + .mem = { + .nregions = N_VHOST_REGIONS, + }, +}; + +/** + * vhost_setup_net() - Open and negotiate features on /dev/vhost-net + * @c: Execution context; c->vhost.fd and c->vhost.features are set + * on success, and left untouched on failure + * + * Failure here means vhost-net isn't usable, not that pasta can't run: the + * caller falls back to plain tap operation unless acceleration was required. + * That's why nothing in here is fatal. + * + * Return: 0 on success, -1 if vhost-net is unavailable or unusable + */ +int vhost_setup_net(struct ctx *c) +{ + const uint64_t req_features = (1ULL << VIRTIO_F_VERSION_1) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t features; + int vhost_fd; + + vhost_fd = open("/dev/vhost-net", O_RDWR | O_NONBLOCK | O_CLOEXEC); + if (vhost_fd < 0) { + debug_perror("Couldn't open /dev/vhost-net"); + return -1; + } + + if (ioctl(vhost_fd, VHOST_SET_OWNER, NULL) < 0) { + debug_perror("VHOST_SET_OWNER ioctl failed"); + goto close_fd; + } + + if (ioctl(vhost_fd, VHOST_GET_FEATURES, &features) < 0) { + debug_perror("VHOST_GET_FEATURES ioctl failed"); + goto close_fd; + } + + if ((features & req_features) != req_features) { + debug("vhost-net is missing features: 0x%016" PRIx64, + req_features & ~features); + goto close_fd; + } + + features = req_features; + if (ioctl(vhost_fd, VHOST_SET_FEATURES, &features) < 0) { + debug_perror("VHOST_SET_FEATURES ioctl failed"); + goto close_fd; + } + + c->vhost.features = features; + c->vhost.fd = vhost_fd; + + return 0; + +close_fd: + close(vhost_fd); + + return -1; +} + +/** + * vhost_setup_eventfds() - Set up one queue's eventfds and ring size + * @c: Execution context; c->vhost.fd must already be set + * @queue_idx: Index of the queue (vring) to configure + * + */ +void vhost_setup_eventfds(struct ctx *c, int queue_idx) +{ + struct vhost_vring_file vring_file = { .index = queue_idx }; + struct vhost_vring_state state = { .index = queue_idx }; + union epoll_ref ref = { 0 }; + int vhost_fd = c->vhost.fd; + int rc; + + state.num = VHOST_NDESCS; + ref.queue = queue_idx; + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost call eventfd, queue %d", + queue_idx); + ref.fd = vring_file.fd; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_CALL, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_CALL ioctl failed, queue %d", + queue_idx); + + ref.type = EPOLL_TYPE_VHOST_CALL; + rc = epoll_add(c->epollfd, EPOLLIN, ref); + if (rc < 0) + die_perror("Failed to watch vhost call eventfd, queue %d", + queue_idx); + c->vhost.vq[queue_idx].call_fd = vring_file.fd; + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost error eventfd, queue %d", + queue_idx); + + rc = ioctl(vhost_fd, VHOST_SET_VRING_ERR, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_ERR ioctl failed, queue %d", + queue_idx); + + ref.type = EPOLL_TYPE_VHOST_ERROR; + ref.fd = vring_file.fd; + rc = epoll_add(c->epollfd, EPOLLIN, ref); + if (rc < 0) + die_perror("Failed to watch vhost error eventfd, queue %d", + queue_idx); + c->vhost.vq[queue_idx].err_fd = vring_file.fd; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_NUM, &state); + if (rc < 0) + die_perror("VHOST_SET_VRING_NUM ioctl failed, queue %d", + queue_idx); + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost kick eventfd, queue %d", + queue_idx); + + rc = ioctl(vhost_fd, VHOST_SET_VRING_KICK, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_KICK ioctl failed, queue %d", + queue_idx); + + c->vhost.vq[queue_idx].kick_fd = vring_file.fd; + + vhost_vq_state[queue_idx].num_free = VHOST_NDESCS; +} + +/** + * vhost_setup_memory_table() - Register the GPA/HVA translation table + * @c: Execution context; c->vhost.fd must already be set + * + * vhost-net reads the addresses we put in descriptors as guest physical + * addresses, and translates them through this table before touching the + * memory they refer to. pasta has no guest and no second address space: + * the addresses we put there are our own virtual addresses. Every region + * below therefore sets guest_phys_addr equal to userspace_addr, so that + * GPA == HVA and the translation is an identity mapping. + * + * Return: 0 on success, -1 on error with errno set + */ +int vhost_setup_memory_table(struct ctx *c) { + size_t region_idx = 0; + + /* general purpose buffers */ + general_register_memory_regions(&vhost_memory, &region_idx); + + /* tcp specific buffers */ + tcp_register_memory_regions(&vhost_memory, &region_idx); + + /* udp specific buffers */ + udp_register_memory_regions(&vhost_memory, &region_idx); + + vhost_memory.mem.nregions = region_idx; + + return ioctl(c->vhost.fd, VHOST_SET_MEM_TABLE, &vhost_memory.mem); +} + +/** + * vhost_set_vring() - Register a vring's addresses and bind its backend + * @c: Execution context; c->vhost.fd must already be set + * @queue_idx: Index of the queue (vring) to configure + * @tap_fd: Tap fd to bind as this queue's backend + */ +void vhost_set_vring(struct ctx *c, int queue_idx, int tap_fd) +{ + int vhost_fd = c->vhost.fd; + struct vhost_vring_addr addr = { + .index = queue_idx, + .desc_user_addr = (unsigned long)vring_desc[queue_idx], + .avail_user_addr = (unsigned long)&vring_avail_all[queue_idx], + .used_user_addr = (unsigned long)&vring_used_all[queue_idx], + .log_guest_addr = (unsigned long)&vring_used_all[queue_idx], + }; + struct vhost_vring_file file = { + .index = queue_idx, + .fd = tap_fd, + }; + unsigned int i; + int rc; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_ADDR, &addr); + if (rc < 0) + die_perror("VHOST_SET_VRING_ADDR ioctl failed, queue %d", + queue_idx); + + if (queue_idx == VHOST_AVAIL_Q_IDX) { + for (i = 0; i < VHOST_NDESCS; ++i) { + vring_desc[VHOST_AVAIL_Q_IDX][i].addr = + (uintptr_t)pkt_buf + i * VHOST_DESC_BYTES; + vring_desc[VHOST_AVAIL_Q_IDX][i].len = VHOST_DESC_BYTES; + vring_desc[VHOST_AVAIL_Q_IDX][i].flags = + VRING_DESC_F_WRITE; + } + + for (i = 0; i < VHOST_NDESCS; ++i) + vring_avail_all[VHOST_AVAIL_Q_IDX].ring[i] = + htole16(i); + + vhost_rx_descriptor_handoff(c); + } + + if (queue_idx == VHOST_USED_Q_IDX) { + for (i = 0; i < (VHOST_NDESCS - 1); ++i) + vring_desc[VHOST_USED_Q_IDX][i].next = i + 1; + } + + rc = ioctl(vhost_fd, VHOST_NET_SET_BACKEND, &file); + if (rc < 0) + die_perror("VHOST_NET_SET_BACKEND ioctl failed, queue %d", + queue_idx); +} + +/** + * vhost_rx_descriptor_handoff() - Announce freed from-guest descriptors + * @c: Execution context + * + * Bumps avail.idx by the number of descriptors accumulated in + * vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free (from prior + * consume_one_rx_descriptor() calls), + * then resets the counter to zero. The kernel will see the new + * avail.idx and consume the freshly-available descriptors. + */ +void vhost_rx_descriptor_handoff(struct ctx *c) +{ + smp_wmb(); + + if (!vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free) + return; + + vring_avail_all[VHOST_AVAIL_Q_IDX].idx += + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free; + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free = 0; + vhost_kick(&vring_used_all[VHOST_AVAIL_Q_IDX], + c->vhost.vq[VHOST_AVAIL_Q_IDX].kick_fd); +} + +/** + * vhost_kick() - Notify the kernel that new descriptors are available + * @used: Used ring of the queue we're announcing on, checked to see + * whether the kernel's virtio thread is already reading + * descriptors and doesn't want to be notified + * @kick_fd: Kick eventfd of that same queue + */ +void vhost_kick(struct vring_used_pasta *used, int kick_fd) +{ + /* Ensure that the read of used->flags doesn't get reordered to be + * above the avail.idx update + */ + smp_mb(); + + if (!(used->flags & VRING_USED_F_NO_NOTIFY)) + eventfd_write(kick_fd, 1); +} diff --git a/vhost.h b/vhost.h new file mode 100644 index 0000000..bd555f9 --- /dev/null +++ b/vhost.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: Ammar Yasser <aerosound161(a)gmail.com> + * + * vhost.h - vhost-net (vhost-kernel) acceleration for pasta mode + */ + +#ifndef VHOST_H +#define VHOST_H + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> + +#include <linux/vhost.h> +#include <linux/vhost_types.h> +#include <linux/virtio_config.h> +#include <linux/virtio_net.h> +#include <linux/virtio_ring.h> + +#include "passt.h" + +#define VHOST_AVAIL_Q_IDX 0 +#define VHOST_USED_Q_IDX 1 + +/** + * struct vq_state - Per-virtqueue local descriptor tracking + * @num_free: Number of descriptors ready to be announced to the + * kernel via vhost_rx_descriptor_handoff() + * @last_used_idx: Number of used-ring entries consumed so far; + * lagging read cursor vs. vring_used->idx (the + * kernel's write cursor) + */ +extern struct vq_state { + uint16_t num_free; + uint16_t last_used_idx; + uint16_t next_free; +} vhost_vq_state[2]; + +struct vring_desc_pasta { + __virtio64 addr; + __virtio32 len; + __virtio16 flags; + __virtio16 next; +}; + +extern struct vring_desc_pasta vring_desc[2][VHOST_NDESCS]; + +struct vring_avail_pasta { + __virtio16 flags; + __virtio16 idx; + __virtio16 ring[VHOST_NDESCS]; +}; +static_assert(sizeof(struct vring_avail_pasta) == + offsetof(struct vring_avail, ring[VHOST_NDESCS]), + "vring_avail_pasta doesn't match the kernel avail ring layout"); +extern struct vring_avail_pasta vring_avail_all[2]; + +struct vring_used_pasta { + __virtio16 flags; + __virtio16 idx; + vring_used_elem_t ring[VHOST_NDESCS]; +}; +static_assert(sizeof(struct vring_used_pasta) == + offsetof(struct vring_used, ring[VHOST_NDESCS]), + "vring_used_pasta doesn't match the kernel used ring layout"); +extern struct vring_used_pasta vring_used_all[2]; + +#define N_VHOST_REGIONS 12 +union vhost_memory_u { + struct vhost_memory mem; + char buf[offsetof(struct vhost_memory, regions[N_VHOST_REGIONS])]; +}; +extern union vhost_memory_u vhost_memory; + +#define VHOST_MEMORY_REGION_PTR(addr, size) \ + (struct vhost_memory_region) { \ + .guest_phys_addr = (uintptr_t)addr, \ + .memory_size = size, \ + .userspace_addr = (uintptr_t)addr, \ + } +#define VHOST_MEMORY_REGION(buf) VHOST_MEMORY_REGION_PTR(&buf, sizeof(buf)) + +void vhost_set_vring(struct ctx *c, int queue_idx, int tap_fd); +int vhost_setup_memory_table(struct ctx *c); +int vhost_setup_net(struct ctx *c); +void vhost_setup_eventfds(struct ctx *c, int queue_idx); +void vhost_rx_descriptor_handoff(struct ctx *c); +void vhost_kick(struct vring_used_pasta *used, int kick_fd); + +#endif /* VHOST_H */ -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 2/7] conf: Add context fields, epoll types and --vhost-kernel flag to pasta
by Ammar Yasser 16 Oct '26

16 Oct '26

--vhost-kernel controls whether to use vhost-net acceleration for pasta. It takes one of three modes: "on" requires acceleration and fails if it isn't available, "off" never uses it, and "auto" uses it if it's available and falls back to plain tap operation otherwise. "auto" is the default, so an unmodified invocation picks up acceleration wherever the kernel offers it, and behaves as before where it doesn't. EPOLL_TYPE_VHOST_CALL means the kernel wants to notify us about data it has written. EPOLL_TYPE_VHOST_ERROR means the kernel encountered an internal error on vhost and wants to tell us something went wrong. Collect the vhost state in a single struct vhost_ctx with the following fields: - mode: which of the three modes above was requested - fd: file descriptor for /dev/vhost-net, which doubles as our indication that setup succeeded - features: virtio feature bits negotiated with vhost-net - vq[]: kick, call and err eventfds for both queues Modify tap_hdr_iov to create an iov with the size of a virtio_net_mrg_rxbuf if vhost initialization has been successful (we are in mode 'on' or 'auto' and we didn't need the tap fallback) Move the L2_MAX_LEN_XX constants from tap.h to passt.h to avoid the circular dependency between both files. VHOST_DESC_BYTES is defined as the maximum l2 frame size + the size of the virtio net header, if the length constant remains in tap.h passt.h would need to include it, but tap.h needs to include passt.h to get the definition of `struct ctx`. Moving the length constants to passt.h makes it free of dependency on tap.h Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- conf.c | 19 +++++++++++++ epoll_type.h | 4 +++ passt.c | 3 ++ passt.h | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++-- tap.h | 36 ++++++------------------ 5 files changed, 112 insertions(+), 29 deletions(-) diff --git a/conf.c b/conf.c index 149a4c7..fe578fc 100644 --- a/conf.c +++ b/conf.c @@ -736,6 +736,9 @@ pasta_opts: " default: auto\n" " --host-lo-to-ns-lo Translate host-loopback forwards to\n" " namespace loopback\n" + " --vhost-kernel[=MODE] Use vhost-kernel acceleration\n" + " MODE is \"on\", \"off\" or \"auto\"\n" + " default: auto\n" " --userns NSPATH Target user namespace to join\n" " --netns PATH|NAME Target network namespace to join\n" " --netns-only Don't join existing user namespace\n" @@ -767,6 +770,7 @@ enum passt_modes conf_mode(int argc, char *argv[]) int vhost_user = 0; const struct option optvu[] = { {"vhost-user", no_argument, &vhost_user, 1 }, + {"vhost-kernel", optional_argument, NULL, 0 }, { 0 }, }; char argv0[PATH_MAX], *basearg0; @@ -1317,6 +1321,7 @@ void conf(struct ctx *c, int argc, char **argv) {"conf-path", required_argument, NULL, 'c' }, {"chroot-fallback", no_argument, NULL, 32 }, {"no-pidns", no_argument, NULL, 33 }, + {"vhost-kernel", optional_argument, NULL, 34 }, { 0 }, }; const char *optstring = "+dqfel:hs:c:F:I:p:P:m:a:n:M:g:i:o:D:S:H:461t:u:T:U:"; @@ -1566,6 +1571,20 @@ void conf(struct ctx *c, int argc, char **argv) die("--no-pidns is for pasta mode only"); c->no_pidns = true; + break; + case 34: + if (c->mode != MODE_PASTA) + die("--vhost-kernel is for pasta mode only"); + + if (!optarg || !strcmp(optarg, "auto")) + c->vhost.mode = VHOST_MODE_AUTO; + else if (!strcmp(optarg, "on")) + c->vhost.mode = VHOST_MODE_ON; + else if (!strcmp(optarg, "off")) + c->vhost.mode = VHOST_MODE_OFF; + else + die("Invalid vhost-kernel mode: %s", optarg); + break; case 'd': c->debug = 1; diff --git a/epoll_type.h b/epoll_type.h index 061325a..e3206d1 100644 --- a/epoll_type.h +++ b/epoll_type.h @@ -50,6 +50,10 @@ enum epoll_type { EPOLL_TYPE_CONF_LISTEN, /* Configuration socket */ EPOLL_TYPE_CONF, + /* vhost-kernel call socket */ + EPOLL_TYPE_VHOST_CALL, + /* vhost-kernel error socket */ + EPOLL_TYPE_VHOST_ERROR, EPOLL_NUM_TYPES, }; diff --git a/passt.c b/passt.c index 5054551..6002231 100644 --- a/passt.c +++ b/passt.c @@ -65,6 +65,7 @@ char pkt_buf[PKT_BUF_BYTES] __attribute__ ((aligned(PAGE_SIZE))); struct ctx passt_ctx = { .pidfile_fd = -1, .fd_tap = -1, + .vhost = { .fd = -1 }, .fd_tap_listen = -1, .fd_control_listen = -1, .fd_repair_listen = -1, @@ -92,6 +93,8 @@ char *epoll_type_str[] = { [EPOLL_TYPE_NL_NEIGH] = "netlink neighbour notifier socket", [EPOLL_TYPE_CONF_LISTEN] = "configuration listening socket", [EPOLL_TYPE_CONF] = "configuration socket", + [EPOLL_TYPE_VHOST_CALL] = "vhost-kernel call socket", + [EPOLL_TYPE_VHOST_ERROR] = "vhost-kernel error socket", }; static_assert(ARRAY_SIZE(epoll_type_str) == EPOLL_NUM_TYPES, "epoll_type_str[] doesn't match enum epoll_type"); diff --git a/passt.h b/passt.h index afd8e9f..be68002 100644 --- a/passt.h +++ b/passt.h @@ -13,8 +13,11 @@ union epoll_ref; #include <stdbool.h> #include <assert.h> +#include <limits.h> #include <sys/epoll.h> +#include <linux/virtio_net.h> + #include "pif.h" #include "packet.h" #include "siphash.h" @@ -35,8 +38,43 @@ union epoll_ref; #define MAC_OUR_LAA \ ((uint8_t [ETH_ALEN]){0x9a, 0x55, 0x9a, 0x55, 0x9a, 0x55}) -/* Large enough for ~128 maximum size frames */ -#define PKT_BUF_BYTES (8UL << 20) + +/** L2_MAX_LEN_PASTA - Maximum frame length for pasta mode (with L2 header) + * + * The kernel tuntap device imposes a maximum frame size of 65535 including + * 'hard_header_len' (14 bytes for L2 Ethernet in the case of "tap" mode). + */ +#define L2_MAX_LEN_PASTA USHRT_MAX + +/** L2_MAX_LEN_PASST - Maximum frame length for passt mode (with L2 header) + * + * The only structural limit the QEMU socket protocol imposes on frames is + * (2^32-1) bytes, but that would be ludicrously long in practice. For now, + * limit it somewhat arbitrarily to 65535 bytes. FIXME: Work out an appropriate + * limit with more precision. + */ +#define L2_MAX_LEN_PASST USHRT_MAX + +/** L2_MAX_LEN_VU - Maximum frame length for vhost-user mode (with L2 header) + * + * vhost-user allows multiple buffers per frame, each of which can be quite + * large, so the inherent frame size limit is rather large. Much larger than is + * actually useful for IP. For now limit arbitrarily to 65535 bytes. FIXME: + * Work out an appropriate limit with more precision. + */ +#define L2_MAX_LEN_VU USHRT_MAX + +/* Number of descriptors in each vhost-net virtqueue */ +#define VHOST_NDESCS 128 + +/* Bytes of pkt_buf backing one from-guest descriptor: a maximum size frame + * plus the virtio-net header the kernel writes in front of it + */ +#define VHOST_DESC_BYTES (L2_MAX_LEN_PASTA + \ + sizeof(struct virtio_net_hdr_mrg_rxbuf)) + +/* One maximum size frame per vhost-net descriptor */ +#define PKT_BUF_BYTES (VHOST_NDESCS * VHOST_DESC_BYTES) extern char pkt_buf [PKT_BUF_BYTES]; @@ -156,6 +194,40 @@ struct ip6_ctx { #include <netinet/if_ether.h> +/** + * enum vhost_mode - Whether to use vhost-kernel acceleration + * @VHOST_MODE_AUTO: Use it if it's available, fall back to plain tap if not + * @VHOST_MODE_ON: Require it, fail if it's not available + * @VHOST_MODE_OFF: Never use it + */ +enum vhost_mode { + VHOST_MODE_AUTO = 0, + VHOST_MODE_ON, + VHOST_MODE_OFF, +}; + +/** + * struct vhost_ctx - Execution context for vhost-kernel acceleration + * @mode: Whether to use acceleration at all, see enum vhost_mode + * @fd: File descriptor for /dev/vhost-net, -1 if not set up + * @features: virtio feature bits negotiated with vhost-net + * @vq: Per-virtqueue eventfds ([0] is from-guest, [1] is to-guest) + * @vq.kick_fd: Written by us, to tell the kernel we queued something + * @vq.call_fd: Written by the kernel, to tell us it queued something + * @vq.err_fd: Written by the kernel on an internal vhost error + */ +struct vhost_ctx { + enum vhost_mode mode; + int fd; + uint64_t features; + + struct { + int kick_fd; + int call_fd; + int err_fd; + } vq[2]; +}; + /** * struct ctx - Execution context * @mode: Operation mode, qemu/UNIX domain socket or namespace/tap @@ -185,6 +257,7 @@ struct ip6_ctx { * @our_tap_mac: Pasta/passt's MAC on the tap link * @guest_mac: MAC address of guest or namespace, seen or configured * @hash_secret: 128-bit secret for siphash functions + * @vhost: vhost-kernel acceleration context, pasta mode only * @ifi4: Template interface for IPv4, -1: none, 0: IPv4 disabled * @ip4: IPv4 configuration * @dns_search: DNS search list @@ -265,6 +338,8 @@ struct ctx { uint64_t hash_secret[2]; + struct vhost_ctx vhost; + int ifi4; struct ip4_ctx ip4; diff --git a/tap.h b/tap.h index 1625975..f8c30e2 100644 --- a/tap.h +++ b/tap.h @@ -12,31 +12,6 @@ #include "passt.h" #include "tap_hdr.h" -/** L2_MAX_LEN_PASTA - Maximum frame length for pasta mode (with L2 header) - * - * The kernel tuntap device imposes a maximum frame size of 65535 including - * 'hard_header_len' (14 bytes for L2 Ethernet in the case of "tap" mode). - */ -#define L2_MAX_LEN_PASTA USHRT_MAX - -/** L2_MAX_LEN_PASST - Maximum frame length for passt mode (with L2 header) - * - * The only structural limit the QEMU socket protocol imposes on frames is - * (2^32-1) bytes, but that would be ludicrously long in practice. For now, - * limit it somewhat arbitrarily to 65535 bytes. FIXME: Work out an appropriate - * limit with more precision. - */ -#define L2_MAX_LEN_PASST USHRT_MAX - -/** L2_MAX_LEN_VU - Maximum frame length for vhost-user mode (with L2 header) - * - * vhost-user allows multiple buffers per frame, each of which can be quite - * large, so the inherent frame size limit is rather large. Much larger than is - * actually useful for IP. For now limit arbitrarily to 65535 bytes. FIXME: - * Work out an appropriate limit with more precision. - */ -#define L2_MAX_LEN_VU USHRT_MAX - struct udphdr; /** @@ -50,10 +25,17 @@ struct udphdr; static inline struct iovec tap_hdr_iov(const struct ctx *c, struct tap_hdr *thdr) { - return (struct iovec){ + struct iovec ret = { .iov_base = thdr, - .iov_len = c->mode == MODE_PASST ? sizeof(*thdr) : 0, }; + + if (c->vhost.fd != -1) { + ret.iov_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); + } else { + ret.iov_len = c->mode == MODE_PASST ? sizeof(thdr->vnet_len) : 0; + } + + return ret; } /** -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 1/7] tap: Move the tap_hdr file to a separate file
by Ammar Yasser 16 Oct '26

16 Oct '26

It was defined in tap.h, put it on its own so that future callers won't need to depend on all definitions in tap.h. Also turn it into a union of a vnet_len and virtio_net_mrg_rxbuf because for vhost acceleration the frames will have this virtio net header prepended to them. Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- tap.h | 9 +-------- tap_hdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 tap_hdr.h diff --git a/tap.h b/tap.h index b335933..1625975 100644 --- a/tap.h +++ b/tap.h @@ -10,6 +10,7 @@ #include <stdint.h> #include "passt.h" +#include "tap_hdr.h" /** L2_MAX_LEN_PASTA - Maximum frame length for pasta mode (with L2 header) * @@ -38,14 +39,6 @@ struct udphdr; -/** - * struct tap_hdr - tap backend specific headers - * @vnet_len: Frame length (for qemu socket transport) - */ -struct tap_hdr { - uint32_t vnet_len; -} __attribute__((packed)); - /** * tap_hdr_iov() - struct iovec for a tap header * @c: Execution context diff --git a/tap_hdr.h b/tap_hdr.h new file mode 100644 index 0000000..4156f44 --- /dev/null +++ b/tap_hdr.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright (c) 2021 Red Hat GmbH + * Author: David Gibson <david(a)gibson.dropbear.id.au> + */ + +#ifndef TAP_HDR_H +#define TAP_HDR_H + +#include <stdint.h> +#include <linux/virtio_net.h> + +/** + * struct tap_hdr - tap backend specific headers + * @vnet_len: Frame length (for qemu socket transport) + */ +struct tap_hdr { + union { + uint32_t vnet_len; + struct virtio_net_hdr_mrg_rxbuf hdr; + }; +}; + +#endif /* TAP_HDR_H */ -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 0/7] Add vhost-net kernel support to pasta
by Ammar Yasser 16 Oct '26

16 Oct '26

This work is based on the original v2 RFC by Eugenio Perez: Link: https://archives.passt.top/passt-dev/20250709174748.3514693-1-eperezma@redh… It is the first finished patch series for vhost-kernel support after a third RFC was submitted: Link: https://archives.passt.top/passt-dev/20260802132155.870796-1-aerosound161@g… vhost-net is a kernel device that allows reading and writing packets to a tap device using virtqueues instead of read(2), write(2), and their vectored counterparts. Frames are handed over through a shared descriptor ring, so several of them can be passed to the kernel at once rather than issuing a syscall per frame. The v3 RFC didn't yield any performance improvements because, to retain correctness, sends between pasta and the vhost-kernel thread had to be synchronous. This patch series fixes this problem through the changes introduced in the last commit, titled: "tap/tcp: Replace tcp_payload_used with a ring buffer style index" The changes reliably yield a roughly 2x speedup on the TX path (container to host). Numbers were obtained through iperf3 with the following methodology and results: Pasta command: ./pasta -f --runas 0:0 --map-guest-addr 192.168.0.5 --vhost-kernel on --config-net 2729785 Host: iperf3 -s1J Container: iperf3 -c 192.168.0.5 -t 10 -P4 -l 1M -w 32M -i1 With --vhost-kernel on: [SUM] 0.00-10.00 sec 1.36 GBytes 1.17 Gbits/sec 3643 sender [SUM] 0.00-10.00 sec 1.16 GBytes 999 Mbits/sec receiver With --vhost-kernel off: [SUM] 0.00-10.00 sec 700 MBytes 587 Mbits/sec 3556 sender [SUM] 0.00-10.10 sec 495 MBytes 411 Mbits/sec receiver v2: - removed all use of epoll_ctl in vhost_setup_eventfds in favor of epoll_add - prevent all functions that register memory regions into the vhost_memory object from overflowing - remove the unions with flexible array members in vring_avail_pasta and vring_used_pasta in favor of hardcoding the ring array to have VHOST_NDESCS entries to avoid having to ignore Wpedantic - remove the && operator with TCP_CURR_IDX when computing no_csum - make desc_id in tx_reap a uint32_t - don't make vhost-kernel a required argument Ammar Yasser (7): tap: Move the tap_hdr file to a separate file conf: Add context fields, epoll types and --vhost-kernel flag to pasta virtio: Add the pasta vhost-net interface and implementation tap: Implement the pasta vhost-net from-guest path tap, tcp, udp: Prepare the to-guest path for vhost-net tap: Implement the pasta vhost-net to-guest path tap/tcp: Replace tcp_payload_used with a ring buffer style index Makefile | 2 +- conf.c | 19 +++ epoll_type.h | 4 + passt.c | 9 ++ passt.h | 79 ++++++++++- tap.c | 368 +++++++++++++++++++++++++++++++++++++++++++++------ tap.h | 47 ++----- tap_hdr.h | 23 ++++ tcp_buf.c | 149 ++++++++++++++++----- tcp_buf.h | 4 + udp.c | 47 +++++-- udp.h | 3 + util.c | 18 +++ util.h | 2 + vhost.c | 295 +++++++++++++++++++++++++++++++++++++++++ vhost.h | 91 +++++++++++++ 16 files changed, 1036 insertions(+), 124 deletions(-) create mode 100644 tap_hdr.h create mode 100644 vhost.c create mode 100644 vhost.h -- 2.39.5 (Apple Git-154)

1 0

[PATCH 0/2] cppcheck 2.22.0 fixes
by Laurent Vivier 15 Oct '26

15 Oct '26

Two fixes for issues detected by cppcheck 2.22.0. The first is a real simplification: a !partial_send guard in tcp_data_from_tap() is always true when reached, because partial_send causes an early return above it, so the condition is dropped. The second is a false positive: cppcheck flags buf[n - 1] as a potential negative index in passt-repair.c, not tracking that n < 0 and n < sizeof(*ev) are both handled before that point. The warning is suppressed with an annotation. Signed-off-by: Laurent Vivier <lvivier(a)redhat.com> Laurent Vivier (2): tcp: Drop always-true !partial_send condition in tcp_data_from_tap() passt-repair: Suppress false positive cppcheck negativeIndex warning passt-repair.c | 1 + tcp.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) -- 2.56.0

1 2

[PASST v10] passt, pasta: Introduce unified multi-address data structures
by Jon Maloy 15 Oct '26

15 Oct '26

As preparation for supporting multiple addresses per interface, we replace the single addr/prefix_len fields with an array. The array consists of a new struct inany_addr_entry containing an address and prefix length, both in inany_addr format. Despite some code refactoring, there are only two real functional changes: - The indicated IPv6 prefix length is now properly stored, instead of being ignored and overridden with the hardcoded value 64, as has been the case until now. - Since even IPv4 addresses now are stored in IPv6 format, we also store the corresponding prefix length in that format, i.e. using the range [96,128] instead of [0,32]. In conf_ip6(), the explicit IN6_IS_ADDR_UNSPECIFIED() check on the address at the end of the function is no longer needed: for host-discovered addresses, it is now checked inside the if (!a) block before calling fwd_set_addr(); for user-provided addresses (via -a), validation already rejects unspecified addresses at parse time. Signed-off-by: Jon Maloy <jmaloy(a)redhat.com> --- v2: -Using inany_addr instead of protocol specific addresses as entry address field. v3: -Merging into one array, directly in struct ctx -Changed prefix_len and flags fields in struct inany_addr_entry to uint8_t, since that makes the struct directly migratable v4: -Updated according to changes in previous commits -Updated according to feedback from David G. -Squashed IP4_MASK macro commit into this one v6: -Renamed and moved some definitions -Introduced fwd_set_addr() and fwd_get_addr() already in this commit -Eliminated first_v4/v6() functions, replaced with fwd_get_addr() -Some other changes as suggested by David G. -I kept the flag CONF_ADDR_LINKLOCAL, since it will be needed later in an address selection function. v7: -Introduced CONF_ADDR_GENERATED flag -Other fixes based on feedback from David and Stefano. -I changed signature of inany_prefix_len(), but I did not change its semantics, since the premise of David's comment is wrong: the caller does *not* explicitly know he is dealing with an IPv4 address. In fact, there are examples later in this series where it may be an IPv6 address, and the caller just trusts he gets the return value in the appropriate format. -Intoduced the inverse of inany_prefix_len(), called inany_prefix_len6() which always returns the prefix in IPv6 or mapped IPv4 format. The name of the function isn't great, but any alternative I came up with became too long to be practical. v8: -Added inany_prefix_v4() (David G.) to extract IPv4 address and prefix length simultaneously, replacing inany_prefix_len(). -Removed both inany_prefix_len() and inany_prefix_len6() auto-detecting functions. Prefix length attached to an inany is now always stored in IPv6 format (96-128 for IPv4-mapped). -fwd_set_addr() now requires prefix_len in IPv6 format; callers updated to pass prefix_len + 96 for IPv4 addresses. -Fixed -n option to store prefix_len in IPv6 format. -Fixed addr_seen regression: when address was set via -a, addr_seen was not initialised in conf_ip4(). -Replaced assert() with graceful no-op in dhcp.c and ndp.c when no address is configured (David G.). -Note: addr_seen is still initialised both at conf_ip*() time and at DHCP/DHCPv6 send time (David's concern). This inconsistency will be addressed in a follow-up. -Introduced a new CONF_ADDR_ANY macro tobe used for address selection, as suggested by Stefano. -Deferred introduction of an explicit scope field (replacing CONF_ADDR_LINKLOCAL flag) to a later commit or series. v9: -Rebased on changes made by David Gibson. -Some minor changes based on feedback from David. -Defined a new macro CONF_ADDR_GLOBAL to supplant CONF_ADDR_ANY. -I did not follow up David's suggestion to introduce three functions to replace the current fwd_get_addr(), because I don't think it would solve any problems. The 'flags' field encode both provenance and type properties orthogonally, and callers need different combinations of them. That is the very point of that field, and I see nothing confusing in it. It is correct that some of the calls to fwd_get_addr() go away in later commits, but enough of them remain to justify its existence. Defining three new functions to replace it would only make them wrappers around fwd_get_addr(), and serve no purpose in my view. v10: -Redefined and refactored function fwd_get_addr(). --- arp.c | 12 ++++- conf.c | 152 ++++++++++++++++++++++++++++++++----------------------- dhcp.c | 20 ++++++-- dhcpv6.c | 15 ++++-- fwd.c | 111 +++++++++++++++++++++++++++++++++------- fwd.h | 4 ++ inany.h | 42 +++++++++++++++ ip.h | 2 + ndp.c | 19 +++++-- passt.h | 60 +++++++++++++++++++--- pasta.c | 23 ++++++--- tap.c | 8 ++- 12 files changed, 358 insertions(+), 110 deletions(-) diff --git a/arp.c b/arp.c index bb042e95..9ee6ae65 100644 --- a/arp.c +++ b/arp.c @@ -41,6 +41,8 @@ static bool ignore_arp(const struct ctx *c, const struct arphdr *ah, const struct arpmsg *am) { + const struct guest_addr *a; + if (ah->ar_hrd != htons(ARPHRD_ETHER) || ah->ar_pro != htons(ETH_P_IP) || ah->ar_hln != ETH_ALEN || @@ -54,7 +56,8 @@ static bool ignore_arp(const struct ctx *c, return true; /* Don't resolve the guest's assigned address, either. */ - if (!memcmp(am->tip, &c->ip4.addr, sizeof(am->tip))) + a = fwd_get_addr(c, AF_INET, true, 0); + if (a && !memcmp(am->tip, inany_v4(&a->addr), sizeof(am->tip))) return true; return false; @@ -123,12 +126,17 @@ int arp(const struct ctx *c, struct iov_tail *data) */ void arp_send_init_req(const struct ctx *c) { + const struct guest_addr *a; struct { struct ethhdr eh; struct arphdr ah; struct arpmsg am; } __attribute__((__packed__)) req; + a = fwd_get_addr(c, AF_INET, true, 0); + if (!a) + return; + /* Ethernet header */ req.eh.h_proto = htons(ETH_P_ARP); memcpy(req.eh.h_dest, MAC_BROADCAST, sizeof(req.eh.h_dest)); @@ -145,7 +153,7 @@ void arp_send_init_req(const struct ctx *c) memcpy(req.am.sha, c->our_tap_mac, sizeof(req.am.sha)); memcpy(req.am.sip, &c->ip4.our_tap_addr, sizeof(req.am.sip)); memcpy(req.am.tha, MAC_BROADCAST, sizeof(req.am.tha)); - memcpy(req.am.tip, &c->ip4.addr, sizeof(req.am.tip)); + memcpy(req.am.tip, inany_v4(&a->addr), sizeof(req.am.tip)); debug("Sending initial ARP request for guest MAC address"); tap_send_single(c, &req, sizeof(req)); diff --git a/conf.c b/conf.c index 149a4c79..6fbdc86d 100644 --- a/conf.c +++ b/conf.c @@ -372,13 +372,16 @@ static uint8_t conf_ip4_prefix(const char *arg) /** * conf_ip4() - Verify or detect IPv4 support, get relevant addresses + * @c: Execution context * @ifi: Host interface to attempt (0 to determine one) - * @ip4: IPv4 context (will be written) * * Return: interface index for IPv4, or 0 on failure. */ -static unsigned int conf_ip4(unsigned int ifi, struct ip4_ctx *ip4) +static unsigned int conf_ip4(struct ctx *c, unsigned int ifi) { + const struct guest_addr *a = fwd_get_addr(c, AF_INET, true, 0); + struct ip4_ctx *ip4 = &c->ip4; + if (!ifi) ifi = nl_get_ext_if(nl_sock, AF_INET); @@ -397,67 +400,69 @@ static unsigned int conf_ip4(unsigned int ifi, struct ip4_ctx *ip4) } } - if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr)) { + if (!a) { + struct in_addr addr; + int prefix_len; int rc = nl_addr_get(nl_sock, ifi, AF_INET, - &ip4->addr, &ip4->prefix_len, NULL); + &addr, &prefix_len, NULL); if (rc < 0) { debug("Couldn't discover IPv4 address: %s", strerror_(-rc)); return 0; } + if (IN4_IS_ADDR_UNSPECIFIED(&addr)) { + warn("Host returned address 0.0.0.0"); + return 0; + } + fwd_set_addr(c, &inany_from_v4(addr), CONF_ADDR_HOST, + prefix_len + 96); } - if (!ip4->prefix_len) { - in_addr_t addr = ntohl(ip4->addr.s_addr); - if (IN_CLASSA(addr)) - ip4->prefix_len = (32 - IN_CLASSA_NSHIFT); - else if (IN_CLASSB(addr)) - ip4->prefix_len = (32 - IN_CLASSB_NSHIFT); - else if (IN_CLASSC(addr)) - ip4->prefix_len = (32 - IN_CLASSC_NSHIFT); - else - ip4->prefix_len = 32; - } - - ip4->addr_seen = ip4->addr; - + a = fwd_get_addr(c, AF_INET, true, 0); + assert(a); + ip4->addr_seen = *inany_v4(&a->addr); ip4->our_tap_addr = ip4->guest_gw; - if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr)) - return 0; - return ifi; } /** * conf_ip4_local() - Configure IPv4 addresses and attributes for local mode - * @ip4: IPv4 context (will be written) + * @c: Execution context (will be written) */ -static void conf_ip4_local(struct ip4_ctx *ip4) +static void conf_ip4_local(struct ctx *c) { - if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr)) - ip4->addr = IP4_LL_GUEST_ADDR; - ip4->addr_seen = ip4->addr; + const struct guest_addr *a = fwd_get_addr(c, AF_INET, true, 0); + struct ip4_ctx *ip4 = &c->ip4; + + if (!a) { + fwd_set_addr(c, &inany_from_v4(IP4_LL_GUEST_ADDR), + CONF_ADDR_GENERATED, + IP4_LL_PREFIX_LEN + 96); + ip4->addr_seen = IP4_LL_GUEST_ADDR; + } else { + ip4->addr_seen = *inany_v4(&a->addr); + } if (IN4_IS_ADDR_UNSPECIFIED(&ip4->guest_gw)) ip4->guest_gw = IP4_LL_GUEST_GW; ip4->our_tap_addr = ip4->guest_gw; - if (!ip4->prefix_len) - ip4->prefix_len = IP4_LL_PREFIX_LEN; - ip4->no_copy_addrs = ip4->no_copy_routes = true; } /** * conf_ip6() - Verify or detect IPv6 support, get relevant addresses + * @c: Execution context * @ifi: Host interface to attempt (0 to determine one) - * @ip6: IPv6 context (will be written) * * Return: interface index for IPv6, or 0 on failure. */ -static unsigned int conf_ip6(unsigned int ifi, struct ip6_ctx *ip6) +static unsigned int conf_ip6(struct ctx *c, unsigned int ifi) { + struct ip6_ctx *ip6 = &c->ip6; + const struct guest_addr *a; + union inany_addr addr; int prefix_len = 0; int rc; @@ -478,23 +483,30 @@ static unsigned int conf_ip6(unsigned int ifi, struct ip6_ctx *ip6) } } - rc = nl_addr_get(nl_sock, ifi, AF_INET6, - IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) ? &ip6->addr : NULL, + rc = nl_addr_get(nl_sock, ifi, AF_INET6, &addr.a6, &prefix_len, &ip6->our_tap_ll); if (rc < 0) { debug("Couldn't discover IPv6 address: %s", strerror_(-rc)); return 0; } - ip6->addr_seen = ip6->addr; + a = fwd_get_addr(c, AF_INET6, true, 0); + if (!a) { + if (IN6_IS_ADDR_UNSPECIFIED(&addr)) + return 0; + + fwd_set_addr(c, &addr, CONF_ADDR_HOST, prefix_len); + ip6->addr_seen = addr.a6; + } else { + ip6->addr_seen = a->addr.a6; + } if (IN6_IS_ADDR_LINKLOCAL(&ip6->guest_gw)) ip6->our_tap_ll = ip6->guest_gw; else ip6->our_tap_addr = ip6->guest_gw; - if (IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) || - IN6_IS_ADDR_UNSPECIFIED(&ip6->our_tap_ll)) + if (IN6_IS_ADDR_UNSPECIFIED(&ip6->our_tap_ll)) return 0; return ifi; @@ -502,10 +514,12 @@ static unsigned int conf_ip6(unsigned int ifi, struct ip6_ctx *ip6) /** * conf_ip6_local() - Configure IPv6 addresses and attributes for local mode - * @ip6: IPv6 context (will be written) + * @c: Execution context (will be written) */ -static void conf_ip6_local(struct ip6_ctx *ip6) +static void conf_ip6_local(struct ctx *c) { + struct ip6_ctx *ip6 = &c->ip6; + if (IN6_IS_ADDR_UNSPECIFIED(&ip6->guest_gw)) ip6->guest_gw = IP6_LL_GUEST_GW; @@ -801,6 +815,7 @@ enum passt_modes conf_mode(int argc, char *argv[]) static void conf_print(const struct ctx *c) { char buf[INANY_ADDRSTRLEN]; + const struct guest_addr *a; int i; if (c->fd_control_listen >= 0) @@ -848,16 +863,19 @@ static void conf_print(const struct ctx *c) inet_ntop(AF_INET, &c->ip4.map_host_loopback, buf, sizeof(buf))); - if (!c->no_dhcp) { + a = fwd_get_addr(c, AF_INET, true, 0); + if (a && !c->no_dhcp) { uint32_t mask; + int plen = 0; - mask = htonl(0xffffffff << (32 - c->ip4.prefix_len)); + inany_prefix_v4(&a->addr, a->prefix_len, &plen); + mask = IN4_MASK(plen); info("DHCP:"); info(" assign: %s", - inet_ntop(AF_INET, &c->ip4.addr, buf, sizeof(buf))); + inany_ntop(&a->addr, buf, sizeof(buf))); info(" mask: %s", - inet_ntop(AF_INET, &mask, buf, sizeof(buf))); + inet_ntop(AF_INET, &mask, buf, sizeof(buf))); info(" router: %s", inet_ntop(AF_INET, &c->ip4.guest_gw, buf, sizeof(buf))); @@ -868,8 +886,8 @@ static void conf_print(const struct ctx *c) break; if (!i) info("DNS:"); - inet_ntop(AF_INET, &c->ip4.dns[i], buf, sizeof(buf)); - info(" %s", buf); + info(" %s", inet_ntop(AF_INET, &c->ip4.dns[i], + buf, sizeof(buf))); } for (i = 0; *c->dns_search[i].n; i++) { @@ -894,13 +912,14 @@ static void conf_print(const struct ctx *c) else goto dns6; - info(" assign: %s", - inet_ntop(AF_INET6, &c->ip6.addr, buf, sizeof(buf))); + a = fwd_get_addr(c, AF_INET6, true, 0); + if (a) + info(" assign: %s", + inany_ntop(&a->addr, buf, sizeof(buf))); info(" router: %s", inet_ntop(AF_INET6, &c->ip6.guest_gw, buf, sizeof(buf))); info(" our link-local: %s", - inet_ntop(AF_INET6, &c->ip6.our_tap_ll, - buf, sizeof(buf))); + inet_ntop(AF_INET6, &c->ip6.our_tap_ll, buf, sizeof(buf))); dns6: for (i = 0; i < ARRAY_SIZE(c->ip6.dns); i++) { @@ -908,8 +927,10 @@ dns6: break; if (!i) info("DNS:"); - inet_ntop(AF_INET6, &c->ip6.dns[i], buf, sizeof(buf)); - info(" %s", buf); + info(" %s", + inet_ntop(AF_INET6, &c->ip6.dns[i], + buf, sizeof(buf))); + } for (i = 0; *c->dns_search[i].n; i++) { @@ -1213,15 +1234,13 @@ static bool conf_addr(struct ctx *c, char *arg, uint8_t opt_n) inany_is_loopback(&addr)) goto bad; + fwd_set_addr(c, &addr, CONF_ADDR_USER, prefix_len); if (a4) { - c->ip4.addr = *a4; - c->ip4.prefix_len = prefix_len - 96; - c->ip4.addr_fixed = true; c->ip4.no_copy_addrs = true; + c->ip4.addr_fixed = true; } else { - c->ip6.addr = addr.a6; - c->ip6.addr_fixed = true; c->ip6.no_copy_addrs = true; + c->ip6.addr_fixed = true; } return is_prefix; @@ -1648,13 +1667,20 @@ void conf(struct ctx *c, int argc, char **argv) case 'a': opt_a_is_prefix = conf_addr(c, optarg, opt_n); break; - case 'n': + case 'n': { + struct guest_addr *a; + if (opt_a_is_prefix) die("Redundant prefix length specification"); - c->ip4.prefix_len = conf_ip4_prefix(optarg); - opt_n = c->ip4.prefix_len + 96; + opt_n = conf_ip4_prefix(optarg) + 96; + + for_each_addr(a, c->addrs, c->addr_count, AF_INET) { + a->prefix_len = opt_n; + break; + } break; + } case 'M': p = optarg; if (!parse_mac(&p, c->our_tap_mac) || @@ -1853,9 +1879,9 @@ void conf(struct ctx *c, int argc, char **argv) nl_sock_init(c, false); if (!v6_only && !c->splice_only) - c->ifi4 = conf_ip4(ifi4, &c->ip4); + c->ifi4 = conf_ip4(c, ifi4); if (!v4_only && !c->splice_only) - c->ifi6 = conf_ip6(ifi6, &c->ip6); + c->ifi6 = conf_ip6(c, ifi6); if (c->ifi4 && c->mtu < IPV4_MIN_MTU) { warn("MTU %"PRIu16" is too small for IPv4 (minimum %u)", @@ -1878,7 +1904,7 @@ void conf(struct ctx *c, int argc, char **argv) if (!c->ifi4 && !v6_only) { if (!c->splice_only) { info("IPv4: no external interface as template, use local mode"); - conf_ip4_local(&c->ip4); + conf_ip4_local(c); } c->ifi4 = -1; } @@ -1886,7 +1912,7 @@ void conf(struct ctx *c, int argc, char **argv) if (!c->ifi6 && !v4_only) { if (!c->splice_only) { info("IPv6: no external interface as template, use local mode"); - conf_ip6_local(&c->ip6); + conf_ip6_local(c); } c->ifi6 = -1; } @@ -1959,7 +1985,7 @@ void conf(struct ctx *c, int argc, char **argv) if (!c->ifi6) { c->no_ndp = 1; c->no_dhcpv6 = 1; - } else if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr)) { + } else if (!fwd_get_addr(c, AF_INET6, true, 0)) { c->no_dhcpv6 = 1; } diff --git a/dhcp.c b/dhcp.c index c3c74224..17eebc4a 100644 --- a/dhcp.c +++ b/dhcp.c @@ -303,7 +303,9 @@ static void opt_set_dns_search(const struct ctx *c, size_t max_len) int dhcp(const struct ctx *c, struct iov_tail *data) { char macstr[ETH_ADDRSTRLEN]; + const struct guest_addr *a; size_t mlen, dlen, opt_len; + const struct in_addr *addr; struct in_addr mask, dst; struct ethhdr eh_storage; struct iphdr iph_storage; @@ -315,6 +317,7 @@ int dhcp(const struct ctx *c, struct iov_tail *data) struct msg const *m; struct msg reply; unsigned int i; + int plen; eh = IOV_REMOVE_HEADER(data, eh_storage); iph = IOV_PEEK_HEADER(data, iph_storage); @@ -344,6 +347,15 @@ int dhcp(const struct ctx *c, struct iov_tail *data) m->op != BOOTREQUEST) return -1; + a = fwd_get_addr(c, AF_INET, true, CONF_ADDR_USER); + if (!a) + a = fwd_get_addr(c, AF_INET, true, 0); + if (!a) + return -1; + + addr = inany_prefix_v4(&a->addr, a->prefix_len, &plen); + assert(addr); + reply.op = BOOTREPLY; reply.htype = m->htype; reply.hlen = m->hlen; @@ -352,7 +364,7 @@ int dhcp(const struct ctx *c, struct iov_tail *data) reply.secs = 0; reply.flags = m->flags; reply.ciaddr = m->ciaddr; - reply.yiaddr = c->ip4.addr; + reply.yiaddr = *addr; reply.siaddr = 0; reply.giaddr = m->giaddr; memcpy(&reply.chaddr, m->chaddr, sizeof(reply.chaddr)); @@ -408,7 +420,7 @@ int dhcp(const struct ctx *c, struct iov_tail *data) info(" from %s", eth_ntop(m->chaddr, macstr, sizeof(macstr))); - mask.s_addr = htonl(0xffffffff << (32 - c->ip4.prefix_len)); + mask.s_addr = IN4_MASK(plen); memcpy(opts[1].s, &mask, sizeof(mask)); memcpy(opts[3].s, &c->ip4.guest_gw, sizeof(c->ip4.guest_gw)); memcpy(opts[54].s, &c->ip4.our_tap_addr, sizeof(c->ip4.our_tap_addr)); @@ -416,7 +428,7 @@ int dhcp(const struct ctx *c, struct iov_tail *data) /* If the gateway is not on the assigned subnet, send an option 121 * (Classless Static Routing) adding a dummy route to it. */ - if ((c->ip4.addr.s_addr & mask.s_addr) + if ((addr->s_addr & mask.s_addr) != (c->ip4.guest_gw.s_addr & mask.s_addr)) { /* a.b.c.d/32:0.0.0.0, 0:a.b.c.d */ opts[121].slen = 14; @@ -475,7 +487,7 @@ int dhcp(const struct ctx *c, struct iov_tail *data) if (m->flags & FLAG_BROADCAST) dst = in4addr_broadcast; else - dst = c->ip4.addr; + dst = *addr; tap_udp4_send(c, c->ip4.our_tap_addr, 67, dst, 68, &reply, dlen); diff --git a/dhcpv6.c b/dhcpv6.c index 29c7e320..5fbba7cf 100644 --- a/dhcpv6.c +++ b/dhcpv6.c @@ -318,7 +318,7 @@ static bool dhcpv6_opt(struct iov_tail *data, uint16_t type) * false otherwise and @data is unmodified */ static bool dhcpv6_ia_notonlink(struct iov_tail *data, - struct in6_addr *la) + const struct in6_addr *la) { int ia_types[2] = { OPT_IA_NA, OPT_IA_TA }; struct opt_ia_addr opt_addr_storage; @@ -567,6 +567,7 @@ int dhcpv6(struct ctx *c, struct iov_tail *data, struct opt_hdr client_id_storage; /* cppcheck-suppress [variableScope,unmatchedSuppression] */ struct opt_ia_na ia_storage; + const struct guest_addr *a; const struct in6_addr *src; struct msg_hdr mh_storage; const struct msg_hdr *mh; @@ -574,6 +575,8 @@ int dhcpv6(struct ctx *c, struct iov_tail *data, const struct udphdr *uh; size_t mlen, n; + a = fwd_get_addr(c, AF_INET6, true, 0); + uh = IOV_REMOVE_HEADER(data, uh_storage); if (!uh) return -1; @@ -627,7 +630,7 @@ int dhcpv6(struct ctx *c, struct iov_tail *data, if (mh->type == TYPE_CONFIRM && server_id) return -1; - if (dhcpv6_ia_notonlink(data, &c->ip6.addr)) { + if (a && dhcpv6_ia_notonlink(data, &a->addr.a6)) { dhcpv6_send_ia_notonlink(c, saddr, data, &client_id_base, @@ -682,7 +685,8 @@ int dhcpv6(struct ctx *c, struct iov_tail *data, resp.hdr.xid = mh->xid; tap_udp6_send(c, src, 547, saddr, 546, mh->xid, &resp, n); - c->ip6.addr_seen = c->ip6.addr; + if (a) + c->ip6.addr_seen = a->addr.a6; return 1; } @@ -694,6 +698,7 @@ int dhcpv6(struct ctx *c, struct iov_tail *data, void dhcpv6_init(const struct ctx *c) { time_t y2k = 946684800; /* Epoch to 2000-01-01T00:00:00Z, no mktime() */ + const struct guest_addr *a; uint32_t duid_time; duid_time = htonl(difftime(time(NULL), y2k)); @@ -706,5 +711,7 @@ void dhcpv6_init(const struct ctx *c) memcpy(resp_not_on_link.server_id.duid_lladdr, c->our_tap_mac, sizeof(c->our_tap_mac)); - resp.ia_addr.addr = c->ip6.addr; + a = fwd_get_addr(c, AF_INET6, true, 0); + if (a) + resp.ia_addr.addr = a->addr.a6; } diff --git a/fwd.c b/fwd.c index 5679a3d6..574db1b3 100644 --- a/fwd.c +++ b/fwd.c @@ -243,6 +243,61 @@ void fwd_neigh_table_init(const struct ctx *c) fwd_neigh_table_update(c, &mga, c->our_tap_mac, true); } +/** + * fwd_set_addr() - Add or update an address in the unified address array + * @c: Execution context + * @addr: Address to add (IPv4-mapped or IPv6) + * @flags: CONF_ADDR_* flags for this address + * @prefix_len: Prefix length in IPv6 format (96-128 for IPv4-mapped) + * + * Find the first existing entry of the same address family and + * overwrite it, or create a new one if none exists + */ +void fwd_set_addr(struct ctx *c, const union inany_addr *addr, + uint8_t flags, int prefix_len) +{ + struct guest_addr *a; + + for_each_addr(a, c->addrs, c->addr_count, inany_af(addr)) { + goto found; + } + + if (c->addr_count >= MAX_GUEST_ADDRS) + return; + + a = &c->addrs[c->addr_count++]; + +found: + a->addr = *addr; + a->prefix_len = prefix_len; + a->flags = flags; +} + +/** + * fwd_get_addr() - Get guest address entry matching criteria + * @c: Execution context + * @af: Address family (AF_INET, AF_INET6, or AF_UNSPEC for any) + * @global: If true, match global only; if false, match link-local only + * @flags: Required provenance flags (any-match), 0 for any provenance + * + * Return: first address entry matching criteria, or NULL + */ +const struct guest_addr *fwd_get_addr(const struct ctx *c, sa_family_t af, + bool global, uint8_t flags) +{ + const struct guest_addr *a; + + for_each_addr(a, c->addrs, c->addr_count, af) { + if (inany_is_linklocal(&a->addr) == global) + continue; + if (flags && !(a->flags & flags)) + continue; + return a; + } + + return NULL; +} + /* Forwarding table storage, generally accessed via pointers in struct ctx */ static struct fwd_table fwd_in; static struct fwd_table fwd_out; @@ -799,8 +854,10 @@ static bool is_dns_flow(uint8_t proto, const struct flowside *ini) * translation, false otherwise */ static bool fwd_guest_accessible4(const struct ctx *c, - const struct in_addr *addr) + const struct in_addr *addr) { + const struct guest_addr *a; + if (IN4_IS_ADDR_LOOPBACK(addr)) return false; @@ -815,7 +872,8 @@ static bool fwd_guest_accessible4(const struct ctx *c, /* For IPv4, addr_seen is initialised to addr, so is always a valid * address */ - if (IN4_ARE_ADDR_EQUAL(addr, &c->ip4.addr) || + a = fwd_get_addr(c, AF_INET, true, 0); + if ((a && IN4_ARE_ADDR_EQUAL(addr, inany_v4(&a->addr))) || IN4_ARE_ADDR_EQUAL(addr, &c->ip4.addr_seen)) return false; @@ -833,10 +891,13 @@ static bool fwd_guest_accessible4(const struct ctx *c, static bool fwd_guest_accessible6(const struct ctx *c, const struct in6_addr *addr) { + const struct guest_addr *a; + if (IN6_IS_ADDR_LOOPBACK(addr)) return false; - if (IN6_ARE_ADDR_EQUAL(addr, &c->ip6.addr)) + a = fwd_get_addr(c, AF_INET6, true, 0); + if (a && IN6_ARE_ADDR_EQUAL(addr, &a->addr.a6)) return false; /* For IPv6, addr_seen starts unspecified, because we don't know what LL @@ -881,16 +942,21 @@ static bool fwd_guest_accessible(const struct ctx *c, static void nat_outbound(const struct ctx *c, const union inany_addr *addr, union inany_addr *translated) { - if (inany_equals4(addr, &c->ip4.map_host_loopback)) + const struct guest_addr *ga; + + if (inany_equals4(addr, &c->ip4.map_host_loopback)) { *translated = inany_loopback4; - else if (inany_equals6(addr, &c->ip6.map_host_loopback)) + } else if (inany_equals6(addr, &c->ip6.map_host_loopback)) { *translated = inany_loopback6; - else if (inany_equals4(addr, &c->ip4.map_guest_addr)) - *translated = inany_from_v4(c->ip4.addr); - else if (inany_equals6(addr, &c->ip6.map_guest_addr)) - translated->a6 = c->ip6.addr; - else + } else if (inany_equals4(addr, &c->ip4.map_guest_addr)) { + ga = fwd_get_addr(c, AF_INET, true, 0); + *translated = ga ? ga->addr : inany_any4; + } else if (inany_equals6(addr, &c->ip6.map_guest_addr)) { + ga = fwd_get_addr(c, AF_INET6, true, 0); + translated->a6 = ga ? ga->addr.a6 : in6addr_any; + } else { *translated = *addr; + } } /** @@ -995,16 +1061,23 @@ bool nat_inbound(const struct ctx *c, const union inany_addr *addr, } else if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.map_host_loopback) && inany_equals6(addr, &in6addr_loopback)) { translated->a6 = c->ip6.map_host_loopback; - } else if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.map_guest_addr) && - inany_equals4(addr, &c->ip4.addr)) { - *translated = inany_from_v4(c->ip4.map_guest_addr); - } else if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.map_guest_addr) && - inany_equals6(addr, &c->ip6.addr)) { - translated->a6 = c->ip6.map_guest_addr; - } else if (fwd_guest_accessible(c, addr)) { - *translated = *addr; } else { - return false; + const struct guest_addr *ga4, *ga6; + + ga4 = fwd_get_addr(c, AF_INET, true, 0); + ga6 = fwd_get_addr(c, AF_INET6, true, 0); + + if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.map_guest_addr) && + ga4 && inany_equals(addr, &ga4->addr)) { + *translated = inany_from_v4(c->ip4.map_guest_addr); + } else if (!IN6_IS_ADDR_UNSPECIFIED(&c->ip6.map_guest_addr) && + ga6 && inany_equals(addr, &ga6->addr)) { + translated->a6 = c->ip6.map_guest_addr; + } else if (fwd_guest_accessible(c, addr)) { + *translated = *addr; + } else { + return false; + } } return true; diff --git a/fwd.h b/fwd.h index b60697d9..d001d8ae 100644 --- a/fwd.h +++ b/fwd.h @@ -79,5 +79,9 @@ void fwd_neigh_table_free(const struct ctx *c, void fwd_neigh_mac_get(const struct ctx *c, const union inany_addr *addr, uint8_t *mac); void fwd_neigh_table_init(const struct ctx *c); +void fwd_set_addr(struct ctx *c, const union inany_addr *addr, + uint8_t flags, int prefix_len); +const struct guest_addr *fwd_get_addr(const struct ctx *c, sa_family_t af, + bool global, uint8_t flags); #endif /* FWD_H */ diff --git a/inany.h b/inany.h index d0befb15..4f2331ee 100644 --- a/inany.h +++ b/inany.h @@ -104,6 +104,34 @@ static inline struct in_addr *inany_v4(const union inany_addr *addr) return (struct in_addr *)&addr->v4mapped.a4; } +/** inany_prefix_v4() - Extract IPv4 address and prefix length from inany + * @addr: IPv4 (as mapped) or IPv6 address + * @prefix_len: Prefix length in IPv6 format (96-128 for IPv4) + * @v4_plen: Set to IPv4 prefix length (0-32) on success + * + * Return: IPv4 address if @addr is IPv4, NULL otherwise + */ +static inline struct in_addr *inany_prefix_v4(const union inany_addr *addr, + int prefix_len, int *v4_plen) +{ + struct in_addr *v4 = inany_v4(addr); + + if (!v4) + return NULL; + *v4_plen = prefix_len - 96; + return v4; +} + +/** inany_af - Get address family of IPv[46] address + * @addr: IPv4 or IPv6 address + * + * Return: AF_INET for IPv4, AF_INET6 for IPv6 + */ +static inline sa_family_t inany_af(const union inany_addr *addr) +{ + return inany_v4(addr) ? AF_INET : AF_INET6; +} + /** inany_default_prefix_len() - Get default prefix length for address * @addr: IPv4 or iPv6 address * @@ -230,6 +258,20 @@ static inline bool inany_is_linklocal6(const union inany_addr *a) return IN6_IS_ADDR_LINKLOCAL(&a->a6); } +/** inany_is_linklocal() - Check if address is link-local + * @a: IPv[46] address + * + * Return: true if @a is in 169.254.0.0/16 or fe80::/10 + */ +static inline bool inany_is_linklocal(const union inany_addr *a) +{ + const struct in_addr *v4 = inany_v4(a); + + if (v4) + return IN4_IS_ADDR_LINKLOCAL(v4); + return IN6_IS_ADDR_LINKLOCAL(&a->a6); +} + /** inany_is_multicast() - Check if address is multicast or broadcast * @a: IPv[46] address * diff --git a/ip.h b/ip.h index aab9b86a..ac93c543 100644 --- a/ip.h +++ b/ip.h @@ -19,6 +19,8 @@ (ntohl(((struct in_addr *)(a))->s_addr) >> IN_CLASSA_NSHIFT == IN_LOOPBACKNET) #define IN4_IS_ADDR_MULTICAST(a) \ (IN_MULTICAST(ntohl(((struct in_addr *)(a))->s_addr))) +#define IN4_MASK(prefix) \ + ((prefix) <= 0 ? 0 : htonl(0xffffffff << (32 - (prefix)))) #define IN4_ARE_ADDR_EQUAL(a, b) \ (((struct in_addr *)(a))->s_addr == ((struct in_addr *)b)->s_addr) #define IN4ADDR_LOOPBACK_INIT \ diff --git a/ndp.c b/ndp.c index 1f2bcb0c..ac1325e1 100644 --- a/ndp.c +++ b/ndp.c @@ -257,7 +257,6 @@ static void ndp_ra(const struct ctx *c, const struct in6_addr *dst) .valid_lifetime = ~0U, .pref_lifetime = ~0U, }, - .prefix = c->ip6.addr, .source_ll = { .header = { .type = OPT_SRC_L2_ADDR, @@ -265,8 +264,15 @@ static void ndp_ra(const struct ctx *c, const struct in6_addr *dst) }, }, }; + const struct guest_addr *a; unsigned char *ptr = NULL; + a = fwd_get_addr(c, AF_INET6, true, 0); + if (!a) + return; + + ra.prefix = a->addr.a6; + ptr = &ra.var[0]; if (c->mtu) { @@ -460,6 +466,7 @@ first: */ void ndp_send_init_req(const struct ctx *c) { + const struct guest_addr *a; struct ndp_ns ns = { .ih = { .icmp6_type = NS, @@ -468,8 +475,14 @@ void ndp_send_init_req(const struct ctx *c) .icmp6_solicited = 0, /* Reserved */ .icmp6_override = 0, /* Reserved */ }, - .target_addr = c->ip6.addr + .target_addr = IN6ADDR_ANY_INIT }; + + a = fwd_get_addr(c, AF_INET6, true, 0); + if (!a) + return; + + ns.target_addr = a->addr.a6; debug("Sending initial NDP NS request for guest MAC address"); - ndp_send(c, &c->ip6.addr, &ns, sizeof(ns)); + ndp_send(c, &a->addr.a6, &ns, sizeof(ns)); } diff --git a/passt.h b/passt.h index afd8e9f0..47e25d5c 100644 --- a/passt.h +++ b/passt.h @@ -64,11 +64,27 @@ enum passt_modes { MODE_VU, }; +/* Maximum number of addresses in context address array */ +#define MAX_GUEST_ADDRS 32 + +/** + * struct guest_addr - Unified IPv4/IPv6 address entry + * @addr: IPv4 (as mapped) or IPv6 address + * @prefix_len: Prefix length in IPv6/IPv4-mapped [0,128]/[96,128] format + * @flags: CONF_ADDR_* flags + */ +struct guest_addr { + union inany_addr addr; + uint8_t prefix_len; + uint8_t flags; +#define CONF_ADDR_USER BIT(0) /* User set via -a */ +#define CONF_ADDR_HOST BIT(1) /* From host interface */ +#define CONF_ADDR_GENERATED BIT(2) /* Generated by PASST/PASTA */ +}; + /** * struct ip4_ctx - IPv4 execution context - * @addr: IPv4 address assigned to guest * @addr_seen: Latest IPv4 address seen as source from tap - * @prefixlen: IPv4 prefix length (netmask) * @guest_gw: IPv4 gateway as seen by the guest * @map_host_loopback: Outbound connections to this address are NATted to the * host's 127.0.0.1 @@ -87,9 +103,7 @@ enum passt_modes { */ struct ip4_ctx { /* PIF_TAP addresses */ - struct in_addr addr; struct in_addr addr_seen; - int prefix_len; struct in_addr guest_gw; struct in_addr map_host_loopback; struct in_addr map_guest_addr; @@ -110,7 +124,6 @@ struct ip4_ctx { /** * struct ip6_ctx - IPv6 execution context - * @addr: IPv6 address assigned to guest * @addr_seen: Latest IPv6 global/site address seen as source from tap * @addr_ll_seen: Latest IPv6 link-local address seen as source from tap * @guest_gw: IPv6 gateway as seen by the guest @@ -132,7 +145,6 @@ struct ip4_ctx { */ struct ip6_ctx { /* PIF_TAP addresses */ - struct in6_addr addr; struct in6_addr addr_seen; struct in6_addr addr_ll_seen; struct in6_addr guest_gw; @@ -192,6 +204,8 @@ struct ip6_ctx { * @fqdn: Guest FQDN * @ifi6: Template interface for IPv6, -1: none, 0: IPv6 disabled * @ip6: IPv6 configuration + * @addrs: Unified address array for both IPv4 (mapped) and IPv6 + * @addr_count: Number of active entries in @addrs array * @pasta_ifn: Name of namespace interface for pasta * @pasta_ifi: Index of namespace interface for pasta * @pasta_conf_ns: Configure namespace after creating it @@ -276,6 +290,9 @@ struct ctx { int ifi6; struct ip6_ctx ip6; + struct guest_addr addrs[MAX_GUEST_ADDRS]; + int addr_count; + char pasta_ifn[IF_NAMESIZE]; unsigned int pasta_ifi; int pasta_conf_ns; @@ -321,6 +338,37 @@ struct ctx { extern struct ctx passt_ctx; +/** + * next_addr_idx_() - Find next address index matching family filter + * @addrs: Array of guest addresses + * @count: Number of addresses in array + * @i: Starting index + * @af: Address family filter: AF_INET, AF_INET6, or AF_UNSPEC for all + * + * Return: next matching index, or count if none found + */ +static inline int next_addr_idx_(const struct guest_addr *addrs, int count, + int i, sa_family_t af) +{ + for (; i < count; i++) { + if (af == AF_UNSPEC || af == inany_af(&addrs[i].addr)) + return i; + } + return i; +} + +/** + * for_each_addr() - Iterate over addresses in array + * @a: Pointer variable for current entry (struct guest_addr *) + * @addrs: Array of guest addresses (e.g., c->addrs) + * @count: Number of addresses (e.g., c->addr_count) + * @af: Address family filter: AF_INET, AF_INET6, or 0 for all + */ +#define for_each_addr(a, addrs, count, af) \ + for (int i_ = next_addr_idx_((addrs), (count), 0, (af)); \ + i_ < (count) && ((a) = &(addrs)[i_], true); \ + i_ = next_addr_idx_((addrs), (count), i_ + 1, (af))) + void proto_update_l2_buf(const unsigned char *eth_d); #endif /* PASST_H */ diff --git a/pasta.c b/pasta.c index 10f3ec8d..24427ddb 100644 --- a/pasta.c +++ b/pasta.c @@ -299,23 +299,32 @@ void pasta_start_ns(struct ctx *c, int argc, char *argv[]) * * Return: 0 on success, negative error code on failure */ -static int pasta_conf_addrs(struct ctx *c, sa_family_t af, int ifi, +static int pasta_conf_addrs(const struct ctx *c, sa_family_t af, int ifi, bool no_copy) { + const struct guest_addr *a; + if (!ifi) return 0; if (!no_copy) return nl_addr_dup(nl_sock, ifi, nl_sock_ns, c->pasta_ifi, af); - if (af == AF_INET && !IN4_IS_ADDR_UNSPECIFIED(&c->ip4.addr)) + a = fwd_get_addr(c, af, true, 0); + if (!a) + return 0; + + if (af == AF_INET) { + const struct in_addr *v4; + int plen = 0; + + v4 = inany_prefix_v4(&a->addr, a->prefix_len, &plen); return nl_addr_set(nl_sock_ns, c->pasta_ifi, AF_INET, - &c->ip4.addr, c->ip4.prefix_len); - if (af == AF_INET6 && !IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr)) - return nl_addr_set(nl_sock_ns, c->pasta_ifi, AF_INET6, - &c->ip6.addr, 64); + v4, plen); + } - return 0; + return nl_addr_set(nl_sock_ns, c->pasta_ifi, AF_INET6, + &a->addr.a6, a->prefix_len); } /** diff --git a/tap.c b/tap.c index dfa66c71..604b54ca 100644 --- a/tap.c +++ b/tap.c @@ -1005,8 +1005,12 @@ resume: c->ip6.addr_seen = *saddr; } - if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr)) - c->ip6.addr = *saddr; + if (!fwd_get_addr(c, AF_INET6, true, 0) && + !fwd_get_addr(c, AF_INET6, false, 0)) { + union inany_addr addr = { .a6 = *saddr }; + + fwd_set_addr(c, &addr, CONF_ADDR_GENERATED, 64); + } } else if (!c->ip6.addr_fixed && !IN6_IS_ADDR_UNSPECIFIED(saddr)) { c->ip6.addr_seen = *saddr; -- 2.52.0

1 0