October 2026 - dev

[PATCH] tcp: Avoid division by zero with small MSS values
by Anshu Kumari 21 Oct '26

21 Oct '26

Fuzz testing with AFL++ persistent mode found a crash in tcp_buf_data_from_sock(), SIGFPE (signal 8) from division by zero at: fill_bufs = DIV_ROUND_UP(wnd_scaled - already_sent, mss); when we split buffers on MSS boundaries, if the MSS is small enough (less than four, rounded down to 0). A guest advertising such a small MSS indicates unusual behaviour because the minimum usable MSS is MTU - IP header - TCP header (28 bytes for the minimum IPv4 MTU of 68). To improve robustness: - Fix MSS_SET() to round up when MSS is between 1 and 3, storing tap_mss as 1 (recovered as 4 by MSS_GET()). - In case of zero MSS, don't send data: tcp_buf_data_from_sock() now returns early. This is not a security issue: the effect of a zero MSS from a guest input was a clean process termination (SIGFPE), not memory corruption or data leakage. Signed-off-by: Anshu Kumari <anskuma(a)redhat.com> --- tcp_buf.c | 3 +++ tcp_conn.h | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tcp_buf.c b/tcp_buf.c index 72c45412..ee8c7ce8 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -335,6 +335,9 @@ int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, struct iovec *iov; uint32_t seq; + if (!mss) + return 0; + /* Set up buffer descriptors we'll fill completely and partially. */ fill_bufs = DIV_ROUND_UP(wnd_scaled - already_sent, mss); if (fill_bufs > TCP_FRAMES) { diff --git a/tcp_conn.h b/tcp_conn.h index e3c55341..3f5ad28b 100644 --- a/tcp_conn.h +++ b/tcp_conn.h @@ -51,7 +51,8 @@ struct tcp_tap_conn { #define TCP_MSS_BITS 14 unsigned int tap_mss :TCP_MSS_BITS; -#define MSS_SET(conn, mss) (conn->tap_mss = (mss >> (16 - TCP_MSS_BITS))) +#define MSS_SET(conn, mss) \ + (conn->tap_mss = (mss) ? MAX(1, (mss) >> (16 - TCP_MSS_BITS)) : 0) #define MSS_GET(conn) (conn->tap_mss << (16 - TCP_MSS_BITS)) #define RTT_EXP_BITS 4 -- 2.55.0

3 3

[PATCH v2] feat: Pass open files to child in pasta mode
by Richard Lawrence 21 Oct '26

21 Oct '26

When pasta mode is used to launch an executable (`pasta [COMMAND]`) and that executable accepts inputs in the form of arbitrary file descriptors (such as `bwrap`), then passt should not stand in the way of the parent process handing off those file descriptors to the child process. See bug 204 for additional discussion. The strategy used here is to delay the closing of inherited file descriptors until after pasta has forked to execute the COMMAND, which happens during the call to `conf()`. A nice side-effect is that the `--fd` argument no longer needs to be parsed early, and can rejoin the other args parsed normally in `conf()`. Highlighted changes: - `conf_tap_fd()` is now called directly by `conf()`. - The tap fd is relocated to a number at least 3, rather than exactly 3. - `snapshot_initial_fds()` memorizes inherited fds early in startup. - Inherited fds are discovered by reading from `/proc` if available. - `isolate_fds()` is now called after `conf()` in `main()`. Signed-off-by: Richard Lawrence <rlawrence(a)tamu.edu> --- conf.c | 39 ++++++++------- conf.h | 1 - isolation.c | 135 ++++++++++++++++++++++++++++++++++++---------------- isolation.h | 9 +++- passt.c | 7 ++- 5 files changed, 129 insertions(+), 62 deletions(-) diff --git a/conf.c b/conf.c index faf2681..6bfdfb6 100644 --- a/conf.c +++ b/conf.c @@ -1169,26 +1169,20 @@ static void conf_sock_listen(const struct ctx *c) } /** - * conf_tap_fd() - Read tap fd as supplied by -F command line option - * @argc: Argument count - * @argv: Command line options + * conf_tap_fd() - Read and relocate tap fd as supplied by -F command line option + * @fdarg: String containing fd + * + * Should: + * - move the --fd descriptor out of the range 0-2 * * Return: fd number from --fd option, or -1 if not supplied */ -int conf_tap_fd(int argc, char **argv) +int conf_tap_fd(const char *fdarg) { - const struct option optfd[] = { { "fd", required_argument, NULL, 'F' }, - { 0 }, }; - const char *fdarg = NULL, *p; unsigned long val; - int name; - - optind = 0; - do { - name = getopt_long(argc, argv, "-:F:", optfd, NULL); - if (name == 'F') - fdarg = optarg; - } while (name != -1); + const char *p; + int new_fd; + int fd; if (!fdarg) return -1; @@ -1197,7 +1191,18 @@ int conf_tap_fd(int argc, char **argv) if (!parse_unsigned(&p, 0, &val) || !parse_eoi(p) || val > INT_MAX) die("Invalid --fd: %s", fdarg); - return val; + fd = (int)val; + if (fd >= 0 && fd < 3) { + new_fd = fcntl(fd, F_DUPFD, 3); + + if (new_fd < 0) + die_perror("Could not relocate --fd descriptor"); + + close(fd); + fd = new_fd; + } + + return fd; } /** @@ -1625,7 +1630,7 @@ void conf(struct ctx *c, int argc, char **argv) c->fd_control_listen = c->fd_control = -1; break; case 'F': - /* --fd was parsed early and c->fd_tap set in main() */ + c->fd_tap = conf_tap_fd(optarg); c->one_off = true; *c->sock_path = 0; break; diff --git a/conf.h b/conf.h index 19bf9bc..16f9718 100644 --- a/conf.h +++ b/conf.h @@ -7,7 +7,6 @@ #define CONF_H enum passt_modes conf_mode(int argc, char *argv[]); -int conf_tap_fd(int argc, char **argv); void conf(struct ctx *c, int argc, char **argv); void conf_listen_handler(struct ctx *c, uint32_t events); void conf_handler(struct ctx *c, uint32_t events); diff --git a/isolation.c b/isolation.c index 94cbe7f..2434748 100644 --- a/isolation.c +++ b/isolation.c @@ -24,19 +24,13 @@ * done anything we need to do with those resources, so we have * multiple stages of self-isolation. In order these are: * - * 1a. isolate_initial() + * 1. isolate_initial() * ==================== * * Executed immediately after startup, drops capabilities we don't * need at any point during execution (or which we gain back when we * need by joining other namespaces). * - * 1b. isolate_fds() - * ================ - * - * Executed immediately after isolate_initial(). Closes any leaked - * files we might have inherited from the parent process. - * * 2. isolate_user() * ================= * @@ -44,14 +38,20 @@ * operate in. Sets our final UID & GID, and enters the correct user * namespace. * - * 3. isolate_prefork() + * 3. isolate_fds() + * ================ + * + * Executed after conf(). Closes any leaked + * files we might have inherited from the parent process. + * + * 4. isolate_prefork() * ==================== * * Executed after all setup, but before daemonising (fork()ing into * the background). Uses mount namespace and pivot_root() to remove * our access to the filesystem. * - * 4. isolate_postfork() + * 5. isolate_postfork() * ===================== * * Executed immediately after daemonizing, but before entering the @@ -61,6 +61,7 @@ * runtime operation. */ +#include <dirent.h> #include <errno.h> #include <fcntl.h> #include <grp.h> @@ -77,6 +78,7 @@ #include <unistd.h> #include <sys/mount.h> #include <sys/prctl.h> +#include <sys/resource.h> #include <sys/socket.h> #include <sys/syscall.h> #include <sys/types.h> @@ -248,47 +250,96 @@ void isolate_initial(void) drop_caps_ep_except(keep); } -/* - * isolate_fds() - Close leaked files, but not --fd, stdin, stdout, stderr - * @argc: Argument count - * @argv: Command line options, as we need to skip any file given via --fd - * - * Should: - * - close all open files except for standard streams and the one from --fd - * - move the --fd descriptor out of the range 0-2 - * - * Return: new fd number for descriptor from --fd, or -1 if not specified +/** + * snapshot_initial_fds() - Snapshot initial file descriptors inherited from parent + * other than standard streams (stdin, stdout, stderr) + * @ifds: Snapshot struct of initial file descriptors to populate */ -int isolate_fds(int argc, char **argv) +void snapshot_initial_fds(struct initial_fd_snapshot *ifds) { - int fd, close_from = STDERR_FILENO + 1; + struct dirent *entry; + size_t capacity = 16; + int max_fd = 1024; + struct rlimit rl; + long parsed_fd; + char *endptr; + int dir_fd; + DIR *dir; + int fd; + + ifds->arr = NULL; + ifds->count = 0; + + dir = opendir("/proc/self/fd"); + if (dir) { + dir_fd = dirfd(dir); + ifds->arr = malloc(sizeof(int) * capacity); + if (!ifds->arr) + die_perror("Failed to allocate memory for inherited fds"); + + while ((entry = readdir(dir)) != NULL) { + if (entry->d_name[0] < '0' || entry->d_name[0] > '9') + continue; + parsed_fd = strtol(entry->d_name, &endptr, 10); + if (*endptr != '\0' || parsed_fd < 0 || parsed_fd > INT_MAX) + continue; + fd = (int)parsed_fd; + /* Ignore stdin/stdout/stderr and the opendir handle itself */ + if (fd > STDERR_FILENO && fd != dir_fd) { + if (ifds->count >= capacity) { + size_t new_cap = capacity * 2; + int *new_arr = realloc(ifds->arr, sizeof(int) * new_cap); + + if (!new_arr) + die_perror("Failed to reallocate memory for inherited fds"); + + ifds->arr = new_arr; + capacity = new_cap; + } + ifds->arr[ifds->count++] = fd; + } + } + closedir(dir); + return; + } + + /* Fallback for environments without /proc (e.g. minimal chroots) */ + if (getrlimit(RLIMIT_NOFILE, &rl) == 0 && rl.rlim_cur != RLIM_INFINITY) + max_fd = (int)rl.rlim_cur; - fd = conf_tap_fd(argc, argv); + ifds->arr = malloc(sizeof(int) * max_fd); + if (!ifds->arr) + die_perror("Failed to allocate memory for inherited fds fallback"); - if (fd >= 0) { - /* Move the passed fd to a more convenient location */ - if (fd != close_from && - (dup2(fd, close_from) != close_from || - close(fd))) - die_perror("Could not move --fd descriptor"); - fd = close_from++; + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) { + if (fcntl(fd, F_GETFD) >= 0) + ifds->arr[ifds->count++] = fd; } +} - if (close_range(close_from, ~0U, CLOSE_RANGE_UNSHARE)) { - if (errno == ENOSYS || errno == EINVAL) { - /* This probably means close_range() or the - * CLOSE_RANGE_UNSHARE flag is not supported by the - * kernel. Not much we can do here except carry on and - * hope for the best. - */ - warn( -"Can't use close_range() to ensure no files leaked by parent"); - } else { - die_perror("Failed to close files leaked by parent"); - } +/** + * isolate_fds() - Close leaked files from the parent process + * @ifds: Snapshot of initial file descriptors + * @keep_fd: File descriptor to keep open, if any + * + * Should: + * - close all file descriptors that were open at startup, except for keep_fd + */ +void isolate_fds(struct initial_fd_snapshot *ifds, int keep_fd) +{ + size_t i; + + if (!ifds || !ifds->arr) + return; + + for (i = 0; i < ifds->count; i++) { + if (ifds->arr[i] != keep_fd) + close(ifds->arr[i]); } - return fd; + free(ifds->arr); + ifds->arr = NULL; + ifds->count = 0; } /** diff --git a/isolation.h b/isolation.h index ec47038..8ada988 100644 --- a/isolation.h +++ b/isolation.h @@ -8,10 +8,17 @@ #define ISOLATION_H #include <stdbool.h> +#include <stddef.h> #include <unistd.h> +struct initial_fd_snapshot { + int *arr; + size_t count; +}; + void isolate_initial(void); -int isolate_fds(int argc, char **argv); +void snapshot_initial_fds(struct initial_fd_snapshot *ifds); +void isolate_fds(struct initial_fd_snapshot *ifds, int keep_fd); void isolate_user(const struct ctx *c, uid_t uid, gid_t gid, bool use_userns, const char *userns); int isolate_prefork(const struct ctx *c); diff --git a/passt.c b/passt.c index 5054551..c4335bf 100644 --- a/passt.c +++ b/passt.c @@ -335,6 +335,7 @@ int main(int argc, char **argv) struct epoll_event events[NUM_EPOLL_EVENTS]; int nfds, devnull_fd = -1, fd; struct ctx *c = &passt_ctx; + struct initial_fd_snapshot ifds; struct rlimit limit; struct timespec now; struct sigaction sa; @@ -344,8 +345,9 @@ int main(int argc, char **argv) arch_avx2_exec(argv); + snapshot_initial_fds(&ifds); + isolate_initial(); - c->fd_tap = isolate_fds(argc, argv); if ((devnull_fd = open("/dev/null", O_RDWR | O_CLOEXEC)) < 0) die_perror("Failed to open /dev/null"); @@ -390,6 +392,9 @@ int main(int argc, char **argv) sock_probe_features(c); conf(c, argc, argv); + + isolate_fds(&ifds, c->fd_tap); + trace_init(c->trace); pasta_netns_quit_init(c); -- 2.52.0

4 8

[PATCH v2 1/3] Fix handling of netlink multipart route dumps.
by Martin Schitter 21 Oct '26

21 Oct '26

This patch adds a `nl_collect` function to store all datagram chunks of a multipart route dump in one single interim buffer for further processing and dependency resolution. --- netlink.c | 79 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/netlink.c b/netlink.c index 225b666..650a6fd 100644 --- a/netlink.c +++ b/netlink.c @@ -229,6 +229,51 @@ static struct nlmsghdr *nl_next(int s, char *buf, struct nlmsghdr *nh, ssize_t * /* NOLINTNEXTLINE(readability-inconsistent-ifelse-braces) */\ } else +/** + * nl_collect() - Collect all respose Datagrams of a netlink multi part dump + * @s: Netlink socket + * @buf: Buffer for responses (at least NLBUFSIZ long) + * @buflen: Buffer size. + * @seq: Sequence number of request we're getting responses for + * + * Return: Length of collected responses in bytes or error code < 0 + */ +int nl_collect(int s, char *buf, size_t buflen, uint32_t seq) +{ + size_t collected = 0; + struct nlmsghdr *nh; + ssize_t n; + int res; + const struct nlmsghdr* tail; + + #define NLMSG_DONE_SIZE NLMSG_LENGTH(4) + + while(collected + NLBUFSIZ < buflen){ + nh = nl_next(s, &buf[collected], NULL, &n); + debug("collect: add chunk of size n=%ld at offset=%ld, seq=%d nlmsg_len=%d", + n, collected, seq, ((const struct nlmsghdr *)&buf[collected])->nlmsg_len); + res = nl_status(nh, n, seq); + if (res < 0) /* error */ + return res; + collected += n; + + /* look for NLMSG_DONE -- it's ussualy transmitted in a seperate datagram, + * but in some rare cases also placed at the end of a multipart chunk.*/ + tail = (const struct nlmsghdr*)&buf[collected - NLMSG_DONE_SIZE]; + if (res == 0 || ( + collected >= NLMSG_DONE_SIZE + && (tail = (const struct nlmsghdr*)&buf[collected - NLMSG_DONE_SIZE]) + && NLMSG_OK(tail, NLMSG_DONE_SIZE) + && (nl_status(tail, NLMSG_DONE_SIZE, seq) == 0)) + ) { + debug("collected: %ld", collected); + return collected; + } + } + err("netlink: Too many routes to duplicate"); + return -E2BIG; +} + /** * nl_do() - Send netlink "do" request, and wait for acknowledgement * @s: Netlink socket @@ -553,21 +598,21 @@ int nl_route_dup(int s_src, unsigned int ifi_src, .rta.rta_len = RTA_LENGTH(sizeof(unsigned int)), .ifi = ifi_src, }; - ssize_t nlmsgs_size, left, status; + ssize_t nlmsgs_size, left, status = 0; unsigned dup_routes = 0; struct nlmsghdr *nh; - char buf[NLBUFSIZ]; + char buf[NLBUFSIZ * 8]; uint32_t seq; unsigned i; seq = nl_send(s_src, &req, RTM_GETROUTE, NLM_F_DUMP, sizeof(req)); - /* nl_foreach() will step through multiple response datagrams, - * which we don't want here because we need to have all the - * routes in the buffer at once. - */ - nh = nl_next(s_src, buf, NULL, &nlmsgs_size); - for (left = nlmsgs_size; + /* collect all the chunks of a multi part dump in one buffer.*/ + nlmsgs_size = nl_collect(s_src, buf, sizeof(buf), seq); + if (nlmsgs_size < 0) + return nlmsgs_size; + + for (nh = (struct nlmsghdr *)buf, left = nlmsgs_size; NLMSG_OK(nh, left) && (status = nl_status(nh, left, seq)) > 0; nh = NLMSG_NEXT(nh, left)) { struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh); @@ -648,24 +693,12 @@ int nl_route_dup(int s_src, unsigned int ifi_src, dup_routes++; } - if (!NLMSG_OK(nh, left)) { - /* Process any remaining datagrams in a different - * buffer so we don't overwrite the first one. - */ - char tail[NLBUFSIZ]; - unsigned extra = 0; - - nl_foreach_oftype(nh, status, s_src, tail, seq, RTM_NEWROUTE) - extra++; - - if (extra) { - err("netlink: Too many routes to duplicate"); - return -E2BIG; - } - } + /* status should be 0 (=NLMSG_DONE) */ if (status < 0) return status; + debug("found %d routes in dump", dup_routes); + /* Routes might have dependencies between each other, and the kernel * processes RTM_NEWROUTE messages sequentially. For n routes, we might * need to send the requests up to n times to get all of them inserted. -- 2.53.0

2 3

[PATCH v4] netlink: Fix nl_route_dup by 3-step processing
by Martin Schitter 21 Oct '26

21 Oct '26

Fixes: #221 Eliminate the route entries capacity limitations of nl_route_dup by copping them to the destination namespace in three successive steps to bypass dependency issues. This approach is taken from the handling used in the restore subcommand of iproute2. Signed-off-by: Martin Schitter <ms+git(a)mur.at> --- netlink.c | 207 ++++++++++++++++++++++++++---------------------------- 1 file changed, 99 insertions(+), 108 deletions(-) diff --git a/netlink.c b/netlink.c index 225b666..803cb20 100644 --- a/netlink.c +++ b/netlink.c @@ -525,6 +525,20 @@ int nl_route_set_def(int s, unsigned int ifi, sa_family_t af, const void *gw) return nl_do(s, &req, RTM_NEWROUTE, NLM_F_CREATE | NLM_F_EXCL, len); } +/** + * nl_rtattr_cmp() -- compare two routing attributes + * @rta1: Routing attribute + * @rta2: Routing attribute + * + * Return: 0 if equal + */ +static int nl_rtattr_cmp(const struct rtattr *rta1, const struct rtattr *rta2) +{ + if (!rta1 || !rta2 || rta1->rta_len != rta2->rta_len) + return 1; + return memcmp(RTA_DATA(rta1), RTA_DATA(rta2), RTA_PAYLOAD(rta1)); +} + /** * nl_route_dup() - Copy routes for given interface and address family * @s_src: Netlink socket in source namespace @@ -535,56 +549,73 @@ int nl_route_set_def(int s, unsigned int ifi, sa_family_t af, const void *gw) * * Return: 0 on success, negative error code on failure */ -int nl_route_dup(int s_src, unsigned int ifi_src, - int s_dst, unsigned int ifi_dst, sa_family_t af) +int nl_route_dup(int s_src, unsigned int ifi_src, int s_dst, + unsigned int ifi_dst, sa_family_t af) { - struct req_t { - struct nlmsghdr nlh; - struct rtmsg rtm; - struct rtattr rta; - unsigned int ifi; - } req = { - .rtm.rtm_family = af, - .rtm.rtm_table = RT_TABLE_MAIN, - .rtm.rtm_scope = RT_SCOPE_UNIVERSE, - .rtm.rtm_type = RTN_UNICAST, + /* Duplicate routes in correct order using 3 succesive stages: + * 0. ones for local addresses, + * 1. ones for local networks, + * 2. others (remote networks/hosts). + */ + for (int prio = 0; prio < 3; prio++) { + struct req_t { + struct nlmsghdr nlh; + struct rtmsg rtm; + struct rtattr rta; + unsigned int ifi; + } req = { + .rtm.rtm_family = af, + .rtm.rtm_table = RT_TABLE_MAIN, + .rtm.rtm_scope = RT_SCOPE_UNIVERSE, + .rtm.rtm_type = RTN_UNICAST, + + .rta.rta_type = RTA_OIF, + .rta.rta_len = RTA_LENGTH(sizeof(unsigned int)), + .ifi = ifi_src, + }; + ssize_t status; + struct nlmsghdr *nh; + char buf[NLBUFSIZ]; + uint32_t seq; + + seq = nl_send(s_src, &req, RTM_GETROUTE, NLM_F_DUMP, + sizeof(req)); + + nl_foreach(nh, status, s_src, buf, seq) { + struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh); + struct rtattr *rta; + size_t na; + uint16_t flags = nh->nlmsg_flags; + int rc; - .rta.rta_type = RTA_OIF, - .rta.rta_len = RTA_LENGTH(sizeof(unsigned int)), - .ifi = ifi_src, - }; - ssize_t nlmsgs_size, left, status; - unsigned dup_routes = 0; - struct nlmsghdr *nh; - char buf[NLBUFSIZ]; - uint32_t seq; - unsigned i; - seq = nl_send(s_src, &req, RTM_GETROUTE, NLM_F_DUMP, sizeof(req)); + if (nh->nlmsg_type != RTM_NEWROUTE) + continue; - /* nl_foreach() will step through multiple response datagrams, - * which we don't want here because we need to have all the - * routes in the buffer at once. - */ - nh = nl_next(s_src, buf, NULL, &nlmsgs_size); - for (left = nlmsgs_size; - NLMSG_OK(nh, left) && (status = nl_status(nh, left, seq)) > 0; - nh = NLMSG_NEXT(nh, left)) { - struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh); - bool discard = false; - struct rtattr *rta; - size_t na; + struct rtattr *tb[RTA_MAX+1] = { 0 }; - if (nh->nlmsg_type != RTM_NEWROUTE) - continue; + for (rta = RTM_RTA(rtm), na = RTM_PAYLOAD(nh); + RTA_OK(rta, na); rta = RTA_NEXT(rta, na)) { + tb[rta->rta_type] = rta; + } - /* nexthop state flags don't apply to freshly created routes, - * and the kernel will refuse our route if they are set. - */ - rtm->rtm_flags &= ~RTNH_COMPARE_MASK; + /* Process only matching routes of current priority + * (variant of an approach used in iproute2s restore_handler) + */ + if (!(prio == 2 && tb[RTA_GATEWAY]) && + !(prio == 1 && !tb[RTA_GATEWAY] && + tb[RTA_PREFSRC] && + nl_rtattr_cmp(tb[RTA_PREFSRC], tb[RTA_DST])) && + !(prio == 0 && !tb[RTA_GATEWAY] && + (!tb[RTA_PREFSRC] || + !nl_rtattr_cmp(tb[RTA_PREFSRC], tb[RTA_DST])))) + continue; + + /* nexthop state flags don't apply to freshly created routes, + * and the kernel will refuse our route if they are set. + */ + rtm->rtm_flags &= ~RTNH_COMPARE_MASK; - for (rta = RTM_RTA(rtm), na = RTM_PAYLOAD(nh); RTA_OK(rta, na); - rta = RTA_NEXT(rta, na)) { /* RTA_OIF and RTA_MULTIPATH attributes carry the * identifier of a host interface. If they match the * host interface we're copying from, change them to @@ -595,103 +626,63 @@ int nl_route_dup(int s_src, unsigned int ifi_src, * available), or if any interface index in nexthop * objects differ from the host interface, discard the * route altogether. - */ - if (rta->rta_type == RTA_OIF) { - if (*(unsigned int *)RTA_DATA(rta) != ifi_src) { - discard = true; - break; + */ + if (tb[RTA_OIF]) { + if (*(unsigned int *)RTA_DATA(tb[RTA_OIF]) != + ifi_src) { + continue; } - - *(unsigned int *)RTA_DATA(rta) = ifi_dst; - } else if (rta->rta_type == RTA_MULTIPATH) { - int nh_len = RTA_PAYLOAD(rta); + *(unsigned int *)RTA_DATA(tb[RTA_OIF]) = + ifi_dst; + } else if (tb[RTA_MULTIPATH]) { + int nh_len = RTA_PAYLOAD(tb[RTA_MULTIPATH]); struct rtnexthop *rtnh; - - for (rtnh = (struct rtnexthop *)RTA_DATA(rta); + bool discard = false; + for (rtnh = (struct rtnexthop *) + RTA_DATA(tb[RTA_MULTIPATH]); RTNH_OK(rtnh, nh_len); - rtnh = RTNH_NEXT_AND_DEC(rtnh, nh_len)) { + rtnh = RTNH_NEXT_AND_DEC(rtnh, + nh_len)) { int src = (int)ifi_src; - if (rtnh->rtnh_ifindex != src) { discard = true; break; } - rtnh->rtnh_ifindex = ifi_dst; } - if (discard) - break; - } else if (rta->rta_type == RTA_PREFSRC || - rta->rta_type == RTA_NH_ID) { + continue; + } else if (tb[RTA_PREFSRC]) { /* Strip RTA_PREFSRC attributes: host routes * might include a preferred source address, * which must be one of the host's addresses. * However, with -a, pasta will use a different * namespace address, making such a route * invalid in the namespace. - * - * Strip RTA_NH_ID attributes: host routes set + */ + tb[RTA_PREFSRC]->rta_type = RTA_UNSPEC; + } else if (tb[RTA_NH_ID]) { + /* Strip RTA_NH_ID attributes: host routes set * up via routing protocols (e.g. OSPF) might * contain a nexthop ID (and not nexthop * objects, which are taken care of in the * RTA_MULTIPATH case above) that's not valid * in the target namespace. */ - rta->rta_type = RTA_UNSPEC; + tb[RTA_NH_ID]->rta_type = RTA_UNSPEC; } - } - - if (discard) - nh->nlmsg_type = NLMSG_NOOP; - else - dup_routes++; - } - - if (!NLMSG_OK(nh, left)) { - /* Process any remaining datagrams in a different - * buffer so we don't overwrite the first one. - */ - char tail[NLBUFSIZ]; - unsigned extra = 0; - - nl_foreach_oftype(nh, status, s_src, tail, seq, RTM_NEWROUTE) - extra++; - - if (extra) { - err("netlink: Too many routes to duplicate"); - return -E2BIG; - } - } - if (status < 0) - return status; - - /* Routes might have dependencies between each other, and the kernel - * processes RTM_NEWROUTE messages sequentially. For n routes, we might - * need to send the requests up to n times to get all of them inserted. - * Routes that have been already inserted will return -EEXIST, but we - * can safely ignore that and repeat the requests. This avoids the need - * to calculate dependencies: let the kernel do that. - */ - for (i = 0; i < dup_routes; i++) { - for (nh = (struct nlmsghdr *)buf, left = nlmsgs_size; - NLMSG_OK(nh, left); - nh = NLMSG_NEXT(nh, left)) { - uint16_t flags = nh->nlmsg_flags; - int rc; - - if (nh->nlmsg_type != RTM_NEWROUTE) - continue; rc = nl_do(s_dst, nh, RTM_NEWROUTE, - (flags & ~NLM_F_DUMP_FILTERED) | NLM_F_CREATE, + (flags & ~NLM_F_DUMP_FILTERED) | + NLM_F_CREATE, nh->nlmsg_len); if (rc < 0 && rc != -EEXIST && rc != -ENETUNREACH && rc != -EHOSTUNREACH) return rc; } + if (status < 0) + return status; } - return 0; } -- 2.53.0

2 1

[PATCH] fwd: Keep listening on remaining rules if one of them fails to bind
by Hayato Kiwata 18 Oct '26

18 Oct '26

fwd_listen_sync_() breaks out of the loop as soon as fwd_sync_one() fails. So, if a single rule fails to bind(), for example because the address isn't assigned to the host, no rule after it in the table is even tried, and those forwards don't work. Reproducer: $ pasta --ns-ifname=tap0 --mtu=65520 --config-net --address=10.0.2.100 \ --netmask=24 --gateway=10.0.2.2 --dns-forward=10.0.2.3 --no-map-gw \ --tcp-ns=none --udp-ns=none --ipv4-only --tcp-ports=none \ --udp-ports=none --conf-path=/tmp/.pasta.sock \ -- python3 -m http.server 80 UNIX domain socket bound at /tmp/.pasta.sock Serving HTTP on 0.0.0.0 port 80 (http://0.0.0.0:80/) ... $ pesto -A --tcp-ports=192.0.2.1/8080:80 /tmp/.pasta.sock $ pesto -A --tcp-ports=127.0.0.1/8085:80 /tmp/.pasta.sock $ pesto -s /tmp/.pasta.sock passt/pasta configuration (/tmp/.pasta.sock) HOST TCP [192.0.2.1]:8080 => 80 TCP [127.0.0.1]:8085 => 80 SPLICE $ curl -sS -o /dev/null localhost:8085 curl: (7) Failed to connect to localhost port 8085 after 0 ms: Could not connect to server pasta only reports the failure for port 8080, and never tries to listen on port 8085: Listen failed for HOST TCP port 192.0.2.1/8080: Cannot assign requested address Couldn't listen on requested ports Error switching to new forwarding rules Keep going through the remaining rules if fwd_sync_one() fails, and still return -1 if any of them failed, as before. Signed-off-by: Hayato Kiwata <dev(a)haytok.jp> --- fwd.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fwd.c b/fwd.c index 5679a3d..6c9df01 100644 --- a/fwd.c +++ b/fwd.c @@ -456,9 +456,8 @@ static int fwd_listen_sync_(void *arg) ns_enter(a->c); for (i = 0; i < a->c->fwd[a->pif]->count; i++) { - a->ret = fwd_sync_one(a->c, a->pif, i, a->tcpmap, a->udpmap); - if (a->ret < 0) - break; + if (fwd_sync_one(a->c, a->pif, i, a->tcpmap, a->udpmap) < 0) + a->ret = -1; } return 0; -- 2.50.1 (Apple Git-155)

1 0

[PATCH v2 0/7] Add vhost-net kernel support to pasta
by aerosouund 16 Oct '26

16 Oct '26

From: Ammar Yasser <aerosound161(a)gmail.com> This work is based on the original v2 RFC by Eugenio Perez: Link: https://archives.passt.top/passt-dev/20250709174748.3514693-1-eperezma@redh… It is the first finished patch series for vhost-kernel support after a third RFC was submitted: Link: https://archives.passt.top/passt-dev/20260802132155.870796-1-aerosound161@g… vhost-net is a kernel device that allows reading and writing packets to a tap device using virtqueues instead of read(2), write(2), and their vectored counterparts. Frames are handed over through a shared descriptor ring, so several of them can be passed to the kernel at once rather than issuing a syscall per frame. The changes reliably yield a roughly 2x speedup on the TX path (container to host). Numbers were obtained through iperf3 with the following methodology and results: Pasta command: ./pasta -f --runas 0:0 --map-guest-addr 192.168.0.5 --vhost-kernel on --config-net 2729785 Host: iperf3 -s1J Container: iperf3 -c 192.168.0.5 -t 10 -P4 -l 1M -w 32M -i1 With --vhost-kernel on: [SUM] 0.00-10.00 sec 1.36 GBytes 1.17 Gbits/sec 3643 sender [SUM] 0.00-10.00 sec 1.16 GBytes 999 Mbits/sec receiver With --vhost-kernel off: [SUM] 0.00-10.00 sec 700 MBytes 587 Mbits/sec 3556 sender [SUM] 0.00-10.10 sec 495 MBytes 411 Mbits/sec receiver v2: - removed all use of epoll_ctl in vhost_setup_eventfds in favor of epoll_add - prevent all functions that register memory regions into the vhost_memory object from overflowing - remove the unions with flexible array members in vring_avail_pasta and vring_used_pasta in favor of hardcoding the ring array to have VHOST_NDESCS entries to avoid having to ignore Wpedantic - remove the && operator with TCP_CURR_IDX when computing no_csum - make desc_id in tx_reap a uint32_t - don't make vhost-kernel a required argument Ammar Yasser (7): tap: Move the tap_hdr file to a separate file conf: Add context fields, epoll types and --vhost-kernel flag to pasta virtio: Add the pasta vhost-net interface and implementation tap: Implement the pasta vhost-net from-guest path tap, tcp, udp: Prepare the to-guest path for vhost-net tap: Implement the pasta vhost-net to-guest path tap/tcp: Replace tcp_payload_used with a ring buffer style index Makefile | 2 +- conf.c | 19 +++ epoll_type.h | 4 + passt.c | 9 ++ passt.h | 79 ++++++++++- tap.c | 368 +++++++++++++++++++++++++++++++++++++++++++++------ tap.h | 47 ++----- tap_hdr.h | 23 ++++ tcp_buf.c | 149 ++++++++++++++++----- tcp_buf.h | 4 + udp.c | 47 +++++-- udp.h | 3 + util.c | 18 +++ util.h | 2 + vhost.c | 295 +++++++++++++++++++++++++++++++++++++++++ vhost.h | 91 +++++++++++++ 16 files changed, 1036 insertions(+), 124 deletions(-) create mode 100644 tap_hdr.h create mode 100644 vhost.c create mode 100644 vhost.h -- 2.39.5 (Apple Git-154)

1 1

[PATCH v2 7/7] tap/tcp: Replace tcp_payload_used with a ring buffer style index
by aerosouund 16 Oct '26

16 Oct '26

From: Ammar Yasser <aerosound161(a)gmail.com> Because pasta resets tcp_payload_used to 0 after every tcp flush this means that pasta will very quickly override data in the tcp buffers after sending. This is not a problem when using writev because the data will have been copied elsewhere by the kernel, but this is a problem for vhost where the kernel thread may not have finished processing this data. This is why a synchronous send was required in vhost, which obviously affects performance. Replace tcp_payload_used with a struct tcp_payload_idx holding the last index we wrote to and a counter that gets updated when we flush packets, from where we last flushed up to that index. Both are grow only and access gets modded by TCP_FRAMES_MEM. Rewrite functions in tcp_buf.c to use that index and handle the wraparound case (sending a batch that started near the end of the buffers and continued to be at the start) in tcp_payload_flush(). With the buffers no longer reused from under the kernel, drop the synchronous wait from tap_send_frames_vhost() and reclaim descriptors until there are enough for the send instead. Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- tap.c | 6 +-- tcp_buf.c | 115 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/tap.c b/tap.c index bc2700f..0456589 100644 --- a/tap.c +++ b/tap.c @@ -490,10 +490,6 @@ static size_t tap_send_frames_vhost(const struct ctx *c, vhost_kick(&vring_used_all[VHOST_USED_Q_IDX], c->vhost.vq[VHOST_USED_Q_IDX].kick_fd); - /* wait until the kernel finishes processing this send */ - while (avail->idx != vring_used_all[1].used.idx) - ; - return processed_frames; } @@ -521,7 +517,7 @@ static size_t tap_send_frames_pasta(const struct ctx *c, size_t i; if (vhost) - return tap_send_frames_vhost(c, iov, bufs_per_frame, nframes); + return tap_send_frames_vhost(c, iov, bufs_per_frame, nframes); for (i = 0; i < nbufs; i += bufs_per_frame) { ssize_t rc = writev(c->fd_tap, iov + i, bufs_per_frame); diff --git a/tcp_buf.c b/tcp_buf.c index 493c5c9..425410f 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -59,7 +59,22 @@ static_assert(MSS6 <= sizeof(tcp_payload[0].data), "MSS6 is greater than 65516") /* References tracking the owner connection of frames in the tap outqueue */ static struct tcp_tap_conn *tcp_frame_conns[TCP_FRAMES_MEM]; -static unsigned int tcp_payload_used; + +/** + * struct tcp_payload_idx - Grow-only cursors into the TCP frame buffers + * @tcp_buf_idx: The last index we wrote data to in the tcp buffers + * @processed: Last index we pushed to the underlying connection + */ +static struct tcp_payload_idx { + unsigned int tcp_buf_idx; + unsigned int processed; +} tcp_idx; + +/* latest index we wrote to into the tcp buffers */ +#define TCP_CURR_IDX (tcp_idx.tcp_buf_idx % TCP_FRAMES_MEM) + +/* current amount of frames queued in tcp buffers but not sent */ +#define TCP_FRAME_COUNT (tcp_idx.tcp_buf_idx - tcp_idx.processed) /* recvmsg()/sendmsg() data for tap */ static struct iovec iov_sock [TCP_FRAMES_MEM + DISCARD_IOV_NUM]; @@ -158,15 +173,46 @@ static void tcp_revert_seq(const struct ctx *c, struct tcp_tap_conn **conns, */ void tcp_payload_flush(const struct ctx *c, const struct timespec *now) { - size_t m; + unsigned int total = TCP_FRAME_COUNT; + unsigned int start = tcp_idx.processed % TCP_FRAMES_MEM; + unsigned int first_batch_size, sent; - m = tap_send_frames(c, &tcp_l2_iov[0][0], TCP_NUM_IOVS, - tcp_payload_used); - if (m != tcp_payload_used) { - tcp_revert_seq(c, &tcp_frame_conns[m], &tcp_l2_iov[m], - tcp_payload_used - m, now); + if (!total) + return; + + /* What is smaller ? all we what we want to send ? or 128 - the index + * indicating the start of where we wrote this batch ? + */ + first_batch_size = MIN(total, TCP_FRAMES_MEM - start); + + sent = tap_send_frames(c, &tcp_l2_iov[start][0], TCP_NUM_IOVS, + first_batch_size); + if (sent < first_batch_size) { + tcp_revert_seq(c, &tcp_frame_conns[start + sent], + &tcp_l2_iov[start + sent], + first_batch_size - sent, now); + goto out; } - tcp_payload_used = 0; + + /* There was more data to send than from tcp_idx.processed up to 128. + * the rest of the batch is going to be at index 0 up total - + * first_batch_size. + */ + if (total > first_batch_size) { + unsigned int second_batch_size = total - first_batch_size; + size_t m2; + + m2 = tap_send_frames(c, &tcp_l2_iov[0][0], TCP_NUM_IOVS, + second_batch_size); + sent += m2; + + if (m2 < second_batch_size) + tcp_revert_seq(c, &tcp_frame_conns[m2], &tcp_l2_iov[m2], + second_batch_size - m2, now); + } + +out: + tcp_idx.processed += sent; } /** @@ -244,13 +290,13 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, uint32_t seq; int ret; - iov = tcp_l2_iov[tcp_payload_used]; + iov = tcp_l2_iov[TCP_CURR_IDX]; if (CONN_V4(conn)) - iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[tcp_payload_used]); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[TCP_CURR_IDX]); else - iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[TCP_CURR_IDX]); - iov[TCP_IOV_ETH] = IOV_OF_LVALUE(tcp_eth_hdr[tcp_payload_used]); + iov[TCP_IOV_ETH] = IOV_OF_LVALUE(tcp_eth_hdr[TCP_CURR_IDX]); payload = iov[TCP_IOV_PAYLOAD].iov_base; seq = conn->seq_to_tap; ret = tcp_prepare_flags(c, conn, flags, &payload->th, @@ -259,7 +305,8 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, if (ret <= 0) return ret; - tcp_frame_conns[tcp_payload_used++] = conn; + tcp_idx.tcp_buf_idx++; + tcp_frame_conns[TCP_CURR_IDX] = conn; l4len = optlen + sizeof(struct tcphdr); iov[TCP_IOV_PAYLOAD].iov_len = l4len; @@ -271,8 +318,9 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, tcp_l2_buf_pad(iov); if (flags & DUP_ACK) { - struct iovec *dup_iov = tcp_l2_iov[tcp_payload_used]; - tcp_frame_conns[tcp_payload_used++] = conn; + struct iovec *dup_iov = tcp_l2_iov[TCP_CURR_IDX]; + tcp_frame_conns[TCP_CURR_IDX] = conn; + tcp_idx.tcp_buf_idx++; memcpy(dup_iov[TCP_IOV_TAP].iov_base, iov[TCP_IOV_TAP].iov_base, iov[TCP_IOV_TAP].iov_len); @@ -284,7 +332,7 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, dup_iov[TCP_IOV_ETH_PAD].iov_len = iov[TCP_IOV_ETH_PAD].iov_len; } - if (tcp_payload_used > TCP_FRAMES_MEM - 2) + if (TCP_FRAME_COUNT > TCP_FRAMES_MEM - 2) tcp_payload_flush(c, now); return 0; @@ -309,21 +357,29 @@ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn, struct iovec *iov; conn->seq_to_tap = seq + dlen; - tcp_frame_conns[tcp_payload_used] = conn; - iov = tcp_l2_iov[tcp_payload_used]; + tcp_frame_conns[TCP_CURR_IDX] = conn; + iov = tcp_l2_iov[TCP_CURR_IDX]; if (CONN_V4(conn)) { if (no_csum) { - struct iovec *iov_prev = tcp_l2_iov[tcp_payload_used - 1]; + /* TCP_CURR_IDX may be zero if the underlying + * tcp_idx.tcp_buf_idx is a multiple of 128, minus one + * will yield an invalid index. The previous index to 0 + * is 127. + */ + unsigned int prev_idx = (TCP_CURR_IDX + + TCP_FRAMES_MEM - 1) % + TCP_FRAMES_MEM; + struct iovec *iov_prev = tcp_l2_iov[prev_idx]; const struct iphdr *iph = iov_prev[TCP_IOV_IP].iov_base; /* overwrite IP4_CSUM flag as we set the checksum */ check = iph->check; } - iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[tcp_payload_used]); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[TCP_CURR_IDX]); } else if (CONN_V6(conn)) { - iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]); + iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[TCP_CURR_IDX]); } - iov[TCP_IOV_ETH].iov_base = &tcp_eth_hdr[tcp_payload_used]; + iov[TCP_IOV_ETH].iov_base = &tcp_eth_hdr[TCP_CURR_IDX]; payload = iov[TCP_IOV_PAYLOAD].iov_base; payload->th.th_off = sizeof(struct tcphdr) / 4; payload->th.th_x2 = 0; @@ -335,7 +391,8 @@ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn, tcp_l2_buf_pad(iov); - if (++tcp_payload_used > TCP_FRAMES_MEM - 1) + tcp_idx.tcp_buf_idx++; + if (TCP_FRAME_COUNT > TCP_FRAMES_MEM - 1) tcp_payload_flush(c, now); } @@ -375,15 +432,13 @@ int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, return -1; } - if (tcp_payload_used + fill_bufs > TCP_FRAMES_MEM) { + if (TCP_FRAME_COUNT + (unsigned int)fill_bufs > TCP_FRAMES_MEM) tcp_payload_flush(c, now); - /* Silence Coverity CWE-125 false positive */ - tcp_payload_used = 0; - } - for (i = 0, iov = iov_sock + DISCARD_IOV_NUM; i < fill_bufs; i++, iov++) { - iov->iov_base = &tcp_payload[tcp_payload_used + i].data; + unsigned int idx = (TCP_CURR_IDX + i) % TCP_FRAMES_MEM; + + iov->iov_base = &tcp_payload[idx].data; iov->iov_len = mss; } if (iov_rem) @@ -457,7 +512,7 @@ int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, dlen = mss; seq = conn->seq_to_tap; for (i = 0; i < send_bufs; i++) { - int no_csum = i && i != send_bufs - 1 && tcp_payload_used; + int no_csum = i && i != send_bufs - 1; bool push = false; if (i == send_bufs - 1) { -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 5/7] tap, tcp, udp: Prepare the to-guest path for vhost-net
by aerosouund 16 Oct '26

16 Oct '26

From: Ammar Yasser <aerosound161(a)gmail.com> Frames sent through vhost-net carry a virtio-net header where pasta's tap header would otherwise go, this necessitates the following two changes: - udp_tap_prepare() has been modified to take a context parameter so it can check if vhost is set up. Only if it isn't should tap_hdr_update be called because this means that the active union variant is vnet_len no the virtio net header - tap_l2_offset() a new helper, has been introduced which tells pcap_multiple() how many bytes precede the L2 frame, which now varies with vhost as well as with the mode tap_send_single() calls tap_send_frames_passt() and tap_send_frames_pasta() directly rather than going through tap_send_frames(). Its callers - ARP, TCP resets, and DHCP, DHCPv6, NDP and ICMP via tap_udp*_send()/tap_icmp*_send() - build their frame in a buffer on the stack, not in the buffers registered with vhost-net. The kernel can only read from registered memory, so these protocols can never take the vhost path and shouldn't be tested for it on every frame. Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- tap.c | 112 ++++++++++++++++++++++++++++++++++-------------------- tcp_buf.c | 12 ++++-- udp.c | 27 +++++++++---- 3 files changed, 99 insertions(+), 52 deletions(-) diff --git a/tap.c b/tap.c index a5fdee8..fd500f1 100644 --- a/tap.c +++ b/tap.c @@ -123,45 +123,6 @@ unsigned long tap_l2_max_len(const struct ctx *c) return 0; /* Unreachable, for cppcheck's sake */ } -/** - * tap_send_single() - Send a single frame - * @c: Execution context - * @data: Packet buffer - * @l2len: Total L2 packet length - */ -void tap_send_single(const struct ctx *c, const void *data, size_t l2len) -{ - uint8_t padded[ETH_ZLEN] = { 0 }; - struct iovec iov[2]; - size_t iovcnt = 0; - uint32_t vnet_len; - - if (l2len < ETH_ZLEN) { - memcpy(padded, data, l2len); - data = padded; - l2len = ETH_ZLEN; - } - - vnet_len = htonl(l2len); - - switch (c->mode) { - case MODE_PASST: - iov[iovcnt] = IOV_OF_LVALUE(vnet_len); - iovcnt++; - /* fall through */ - case MODE_PASTA: - iov[iovcnt].iov_base = (void *)data; - iov[iovcnt].iov_len = l2len; - iovcnt++; - - tap_send_frames(c, iov, iovcnt, 1); - break; - case MODE_VU: - vu_send_single(c, data, l2len); - break; - } -} - /** * tap_push_l2h() - Build an L2 header for an inbound packet * @c: Execution context @@ -497,6 +458,76 @@ static size_t tap_send_frames_passt(const struct ctx *c, return i / bufs_per_frame; } +/** + * tap_l2_offset() - Backend specific header size preceding each L2 frame + * @c: Execution context + * + * Return: offset of the L2 frame within each frame's first buffer + */ +static size_t tap_l2_offset(const struct ctx *c) +{ + if (c->mode == MODE_PASST) + return sizeof(uint32_t); /* vnet_len */ + + if (c->vhost.fd != -1) + return VNET_HLEN; + + return 0; +} + +/** + * tap_send_single() - Send a single frame + * @c: Execution context + * @data: Packet buffer + * @l2len: Total L2 packet length + */ +void tap_send_single(const struct ctx *c, const void *data, size_t l2len) +{ + uint8_t padded[ETH_ZLEN] = { 0 }; + struct iovec iov[2]; + uint32_t vnet_len; + size_t iovcnt = 0; + size_t m = 0; + + if (l2len < ETH_ZLEN) { + memcpy(padded, data, l2len); + data = padded; + l2len = ETH_ZLEN; + } + + vnet_len = htonl(l2len); + + switch (c->mode) { + case MODE_PASST: + /* create an iov for the length */ + iov[iovcnt] = IOV_OF_LVALUE(vnet_len); + iovcnt++; + /* create the data iov */ + iov[iovcnt].iov_base = (void *)data; + iov[iovcnt].iov_len = l2len; + iovcnt++; + + m = tap_send_frames_passt(c, iov, iovcnt, 1); + break; + case MODE_PASTA: + /* don't create a length iov in the case of pasta */ + iov[iovcnt].iov_base = (void *)data; + iov[iovcnt].iov_len = l2len; + iovcnt++; + + m = tap_send_frames_pasta(c, iov, iovcnt, 1); + break; + case MODE_VU: + vu_send_single(c, data, l2len); + return; + } + + if (!m) + debug("tap: failed to send a single frame"); + + pcap_multiple(iov, iovcnt, m, tap_l2_offset(c)); +} + /** * tap_send_frames() - Send out multiple prepared frames * @c: Execution context @@ -537,8 +568,7 @@ size_t tap_send_frames(const struct ctx *c, const struct iovec *iov, debug("tap: failed to send %zu frames of %zu", nframes - m, nframes); - pcap_multiple(iov, bufs_per_frame, m, - c->mode == MODE_PASST ? sizeof(uint32_t) : 0); + pcap_multiple(iov, bufs_per_frame, m, tap_l2_offset(c)); return m; } diff --git a/tcp_buf.c b/tcp_buf.c index dd80384..493c5c9 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -21,6 +21,7 @@ #include <netinet/ip.h> #include <netinet/tcp.h> +#include <linux/virtio_net.h> #include "util.h" #include "ip.h" @@ -44,7 +45,7 @@ /* Ethernet header for IPv4 and IPv6 frames */ static struct ethhdr tcp_eth_hdr[TCP_FRAMES_MEM]; -static struct tap_hdr tcp_payload_tap_hdr[TCP_FRAMES_MEM]; +static struct virtio_net_hdr_mrg_rxbuf tcp_payload_tap_hdr[TCP_FRAMES_MEM]; /* IP headers for IPv4 and IPv6 */ static struct iphdr tcp4_payload_ip[TCP_FRAMES_MEM]; @@ -113,7 +114,7 @@ void tcp_sock_iov_init(const struct ctx *c) for (i = 0; i < TCP_FRAMES_MEM; i++) { struct iovec *iov = tcp_l2_iov[i]; - iov[TCP_IOV_TAP] = tap_hdr_iov(c, &tcp_payload_tap_hdr[i]); + iov[TCP_IOV_TAP] = tap_hdr_iov(c, (struct tap_hdr *)&tcp_payload_tap_hdr[i]); iov[TCP_IOV_ETH].iov_len = sizeof(struct ethhdr); iov[TCP_IOV_PAYLOAD].iov_base = &tcp_payload[i]; iov[TCP_IOV_ETH_PAD].iov_base = eth_pad; @@ -216,7 +217,12 @@ static void tcp_l2_buf_fill_headers(const struct ctx *c, l2len = tcp_fill_headers(c, conn, eh, ip4h, ip6h, th, &tail, iov_tail_size(&tail), csum_flags, seq); - tap_hdr_update(taph, l2len); + + /* With vhost-net this buffer holds a virtio-net header, which a tap + * one must not be written over + */ + if (c->vhost.fd == -1) + tap_hdr_update(taph, l2len); } /** diff --git a/udp.c b/udp.c index a4355a1..afe1c7f 100644 --- a/udp.c +++ b/udp.c @@ -103,6 +103,7 @@ #include <time.h> #include <arpa/inet.h> #include <linux/errqueue.h> +#include <linux/virtio_net.h> #include "checksum.h" #include "util.h" @@ -149,12 +150,16 @@ static struct ethhdr udp_eth_hdr[UDP_MAX_FRAMES]; * struct udp_meta_t - Pre-cooked headers for UDP packets * @ip6h: Pre-filled IPv6 header (except for payload_len and addresses) * @ip4h: Pre-filled IPv4 header (except for tot_len and saddr) - * @taph: Tap backend specific header + * @vnet_hdr: virtio-net header, used when sending through vhost-net + * @taph: Tap backend specific header, used otherwise */ static struct udp_meta_t { struct ipv6hdr ip6h; struct iphdr ip4h; - struct tap_hdr taph; + union { + struct virtio_net_hdr_mrg_rxbuf vnet_hdr; + struct tap_hdr taph; + }; } #ifdef __AVX2__ __attribute__ ((aligned(32))) @@ -356,13 +361,15 @@ static void udp_tap_pad(struct iovec *iov) /** * udp_tap_prepare() - Convert one datagram into a tap frame + * @c: Execution context * @mmh: Receiving mmsghdr array * @idx: Index of the datagram to prepare * @tap_omac: MAC address of remote endpoint as seen from the guest * @toside: Flowside for destination side * @no_udp_csum: Do not set UDP checksum */ -static void udp_tap_prepare(const struct mmsghdr *mmh, +static void udp_tap_prepare(const struct ctx *c, + const struct mmsghdr *mmh, unsigned int idx, const uint8_t *tap_omac, const struct flowside *toside, @@ -384,8 +391,10 @@ static void udp_tap_prepare(const struct mmsghdr *mmh, udp_update_hdr6(&bm->ip6h, uh, &payload, toside, mmh[idx].msg_len, no_udp_csum); - l2len = MAX(l4len + sizeof(bm->ip6h) + ETH_HLEN, ETH_ZLEN); - tap_hdr_update(&bm->taph, l2len); + if (c->mode == MODE_PASST) { + l2len = MAX(l4len + sizeof(bm->ip6h) + ETH_HLEN, ETH_ZLEN); + tap_hdr_update(&bm->taph, l2len); + } eh->h_proto = htons_constant(ETH_P_IPV6); (*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip6h); @@ -393,8 +402,10 @@ static void udp_tap_prepare(const struct mmsghdr *mmh, udp_update_hdr4(&bm->ip4h, uh, &payload, toside, mmh[idx].msg_len, no_udp_csum); - l2len = MAX(l4len + sizeof(bm->ip4h) + ETH_HLEN, ETH_ZLEN); - tap_hdr_update(&bm->taph, l2len); + if (c->mode == MODE_PASST) { + l2len = MAX(l4len + sizeof(bm->ip4h) + ETH_HLEN, ETH_ZLEN); + tap_hdr_update(&bm->taph, l2len); + } eh->h_proto = htons_constant(ETH_P_IP); (*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip4h); @@ -867,7 +878,7 @@ static void udp_buf_sock_to_tap(const struct ctx *c, int s, int n, fwd_neigh_mac_get(c, &toside->oaddr, omac); for (i = 0; i < n; i++) - udp_tap_prepare(udp_mh_recv, i, omac, toside, false); + udp_tap_prepare(c, udp_mh_recv, i, omac, toside, false); tap_send_frames(c, &udp_l2_iov[0][0], UDP_NUM_IOVS, n); } -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 4/7] tap: Implement the pasta vhost-net from-guest path
by aerosouund 16 Oct '26

16 Oct '26

From: Ammar Yasser <aerosound161(a)gmail.com> Add tap_vhost_input(), called when the kernel signals on a call eventfd that it has written guest frames into the from-guest queue. It consumes descriptors with consume_one_rx_descriptor(), skips the virtio_net header on each, builds an iov_tail from what's left and queues it for processing, then tells the kernel the descriptors are free to reuse. consume_one_rx_descriptor() pops one entry from the used ring, advances our read cursor and the count of descriptors awaiting handoff, and returns a pointer to that descriptor's data. Bootstrap vhost in tap_sock_tun_init(). If setup fails we leave c->vhost.fd at -1 and carry on unaccelerated. With vhost-net running the kernel reads the tap fd itself, so don't add that fd to our epoll set at all: tap_handler_pasta() would otherwise be woken for data it must not read. Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- passt.c | 6 +++ tap.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- tap.h | 2 + 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/passt.c b/passt.c index 6002231..372b743 100644 --- a/passt.c +++ b/passt.c @@ -305,6 +305,12 @@ static void passt_worker(void *opaque, int nfds, struct epoll_event *events) case EPOLL_TYPE_CONF: conf_handler(c, eventmask); break; + case EPOLL_TYPE_VHOST_CALL: + tap_vhost_input(c, ref, &now); + break; + case EPOLL_TYPE_VHOST_ERROR: + die("Error on vhost-kernel socket"); + break; default: /* Can't happen */ assert(0); diff --git a/tap.c b/tap.c index dfa66c7..a5fdee8 100644 --- a/tap.c +++ b/tap.c @@ -38,6 +38,7 @@ #include <netinet/udp.h> #include <netinet/ip_icmp.h> #include <netinet/if_ether.h> +#include <sys/eventfd.h> #include <linux/if_tun.h> #include <linux/icmpv6.h> @@ -61,6 +62,7 @@ #include "vhost_user.h" #include "vu_common.h" #include "epoll_ctl.h" +#include "vhost.h" /* Maximum allowed frame lengths (including L2 header) */ @@ -1446,7 +1448,11 @@ static void tap_start_connection(const struct ctx *c) break; } - epoll_add(c->epollfd, EPOLLIN | EPOLLRDHUP, ref); + /* With vhost-net running, the kernel reads the tap fd for us and + * notifies us on the call eventfd, so we mustn't watch it ourselves + */ + if (c->vhost.fd == -1) + epoll_add(c->epollfd, EPOLLIN | EPOLLRDHUP, ref); if (!tap_is_ready(c)) return; @@ -1514,6 +1520,89 @@ void tap_listen_handler(struct ctx *c, uint32_t events) tap_start_connection(c); } +/** + * consume_one_rx_descriptor() - Consume one used from-guest descriptor + * @len: Set to the length of data written by the kernel + * + * Pops a single entry from the used ring, advancing our read cursor and the + * count of descriptors awaiting refill announcement. + * + * NOTE: This function assumes the kernel is going to post single descriptors + * always, No chains. If that changes, we would need to increment num_free + * as we advertise back to the kernel the free descriptors by the length of the chain. + * + * Return: pointer to the packet buffer, or NULL if no data is available + */ +static void *consume_one_rx_descriptor(size_t *len) +{ + struct vring_used_pasta *used = &vring_used_all[VHOST_AVAIL_Q_IDX]; + uint16_t used_idx, last_used; + uint32_t i; + + used_idx = le16toh(used->idx); + + smp_rmb(); + + /* If the kernel's used index matches ours, we've consumed everything + * it has posted + */ + if (used_idx == vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx) { + *len = 0; + return NULL; + } + + last_used = vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx % VHOST_NDESCS; + i = le32toh(used->ring[last_used].id); + *len = le32toh(used->ring[last_used].len); + + if (i != last_used) + die("vhost: id %u at used position %u", i, last_used); + + /* The kernel has queued for us something we cannot receive */ + if (*len > VHOST_DESC_BYTES) + die("vhost: id %u len %zu > %zu", i, *len, + (size_t)VHOST_DESC_BYTES); + + vhost_vq_state[VHOST_AVAIL_Q_IDX].last_used_idx++; + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free++; + + return pkt_buf + i * VHOST_DESC_BYTES; +} + +/** + * tap_vhost_input() - Handle frames the kernel wrote to the from-guest queue + * @c: Execution context + * @ref: epoll reference + * @now: Current timestamp + */ +void tap_vhost_input(struct ctx *c, union epoll_ref ref, + const struct timespec *now) +{ + struct virtio_net_hdr_mrg_rxbuf *hdr; + struct iov_tail data; + size_t len; + + eventfd_read(ref.fd, (eventfd_t[]){ 0 }); + + tap_flush_pools(); + + while ((hdr = consume_one_rx_descriptor(&len))) { + /* a valid receive is one where there is a virtio net header followed by atleast sizeof(struct ethhdr) bytes */ + if (len < (sizeof(*hdr) + sizeof(struct ethhdr))) { + warn("vhost: invalid len %lu", len); + continue; + } + + /* Skip over the vnet header, add the packet without it */ + data = IOV_TAIL_FROM_BUF((void *)(hdr + 1), + len - sizeof(*hdr), 0); + tap_add_packet(c, &data, now); + } + + tap_handler(c, now); + vhost_rx_descriptor_handoff(c); +} + /** * tap_ns_tun() - Get tuntap fd in namespace * @c: Execution context @@ -1555,12 +1644,33 @@ static int tap_ns_tun(void *arg) */ static void tap_sock_tun_init(struct ctx *c) { + int i; + if (!c->splice_only) { NS_CALL(tap_ns_tun, c); if (c->fd_tap == -1) die("Failed to set up tap device in namespace"); } + /* Nothing else here runs if this fails: c->vhost.fd stays -1 and we + * carry on reading the tap fd ourselves, unless acceleration was + * explicitly asked for + */ + if (c->vhost.mode != VHOST_MODE_OFF && vhost_setup_net(c) < 0 && + c->vhost.mode == VHOST_MODE_ON) + die("vhost-net is not available"); + + if (c->vhost.fd != -1) { + for (i = 0; i < ARRAY_SIZE(c->vhost.vq); i++) + vhost_setup_eventfds(c, i); + + if (vhost_setup_memory_table(c) < 0) + die_perror("VHOST_SET_MEM_TABLE ioctl failed"); + + for (i = 0; i < ARRAY_SIZE(c->vhost.vq); i++) + vhost_set_vring(c, i, c->fd_tap); + } + pasta_ns_conf(c); if (!c->splice_only) diff --git a/tap.h b/tap.h index f8c30e2..0f1f7a3 100644 --- a/tap.h +++ b/tap.h @@ -48,6 +48,8 @@ static inline void tap_hdr_update(struct tap_hdr *thdr, size_t l2len) thdr->vnet_len = htonl(l2len); } +void tap_vhost_input(struct ctx *c, union epoll_ref ref, + const struct timespec *now); unsigned long tap_l2_max_len(const struct ctx *c); void *tap_push_l2h(const struct ctx *c, void *buf, const void *src_mac, uint16_t proto); -- 2.39.5 (Apple Git-154)

1 0

[PATCH v2 3/7] virtio: Add the pasta vhost-net interface and implementation
by aerosouund 16 Oct '26

16 Oct '26

From: Ammar Yasser <aerosound161(a)gmail.com> Add vhost.c and vhost.h which define the api and implementation for pasta vhost acceleration - vq_state/vhost_vq_state[]: our own view of each virtqueue, which the kernel doesn't track for us: descriptors ready to hand back, how far we've read the used ring, and the next descriptor to allocate - vring_desc, vring_avail_all, vring_used_all, vhost_memory: the rings and the region table shared with the kernel - vhost_setup_net: open /dev/vhost-net and negotiate VIRTIO_F_VERSION_1 and VHOST_NET_F_VIRTIO_NET_HDR. Failure is fatal only if --vhost-kernel is set to 'on'. - vhost_setup_eventfds: create, register and watch the call, kick and error eventfds for one queue - vhost_setup_memory_table: register the buffers the kernel may touch. pasta has no guest, so each region maps guest_phys_addr onto the identical userspace_addr, making GPA translation an identity mapping. This function uses the *_register_memory_regions functions created per protocol and for general purpose buffers. - vhost_set_vring: publish a vring's addresses, bind the tap fd as its backend and lay out its descriptors - vhost_rx_descriptor_handoff, vhost_kick: hand consumed descriptors back and notify the kernel unless it says it's already polling Signed-off-by: Eugenio Pérez <eperezma(a)redhat.com> Signed-off-by: Ammar Yasser <aerosound161(a)gmail.com> --- Makefile | 2 +- tcp_buf.c | 22 +++- tcp_buf.h | 4 + udp.c | 20 ++++ udp.h | 3 + util.c | 18 ++++ util.h | 2 + vhost.c | 295 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ vhost.h | 91 +++++++++++++++++ 9 files changed, 455 insertions(+), 2 deletions(-) create mode 100644 vhost.c create mode 100644 vhost.h diff --git a/Makefile b/Makefile index f6c87b1..6f11e4b 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ PASST_SRCS = arch.c arp.c bitmap.c checksum.c conf.c dhcp.c dhcpv6.c \ isolation.c lineread.c log.c mld.c ndp.c netlink.c migrate.c packet.c \ parse.c passt.c pasta.c pcap.c pif.c repair.c serialise.c tap.c tcp.c \ tcp_buf.c tcp_splice.c tcp_vu.c udp.c udp_flow.c udp_vu.c util.c \ - vhost_user.c virtio.c vu_common.c + vhost.c vhost_user.c virtio.c vu_common.c PASST_REPAIR_SRCS = passt-repair.c PESTO_SRCS = pesto.c bitmap.c fwd_rule.c inany.c ip.c lineread.c parse.c \ serialise.c diff --git a/tcp_buf.c b/tcp_buf.c index 72c4541..dd80384 100644 --- a/tcp_buf.c +++ b/tcp_buf.c @@ -31,12 +31,14 @@ #include "inany.h" #include "tcp_conn.h" #include "tcp_internal.h" -#include "tcp_buf.h" +#include "vhost.h" #define TCP_FRAMES_MEM 128 #define TCP_FRAMES \ (c->mode == MODE_PASTA ? 1 : TCP_FRAMES_MEM) +#define TCP_NREGIONS 5 + /* Static buffers */ /* Ethernet header for IPv4 and IPv6 frames */ @@ -75,6 +77,24 @@ void tcp_update_l2_buf(const unsigned char *eth_d) eth_update_mac(&tcp_eth_hdr[i], eth_d, NULL); } +/** + * tcp_register_memory_region() - Register the TCP specific buffers into a memory + * struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void tcp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + TCP_NREGIONS) > N_VHOST_REGIONS) + die("tcp: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + TCP_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_payload_tap_hdr); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp4_payload_ip); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp6_payload_ip); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_payload); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(tcp_eth_hdr); +} + /** * tcp_sock_iov_init() - Initialise scatter-gather L2 buffers for IPv4 sockets * @c: Execution context diff --git a/tcp_buf.h b/tcp_buf.h index 5d31cea..5e341ed 100644 --- a/tcp_buf.h +++ b/tcp_buf.h @@ -6,11 +6,15 @@ #ifndef TCP_BUF_H #define TCP_BUF_H +union vhost_memory_u; +struct tcp_tap_conn; + void tcp_sock_iov_init(const struct ctx *c); void tcp_payload_flush(const struct ctx *c, const struct timespec *now); int tcp_buf_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn, uint32_t already_sent, const struct timespec *now); int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags, const struct timespec *now); +void tcp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); #endif /*TCP_BUF_H */ diff --git a/udp.c b/udp.c index 505e554..a4355a1 100644 --- a/udp.c +++ b/udp.c @@ -118,6 +118,7 @@ #include "udp_internal.h" #include "udp_vu.h" #include "epoll_ctl.h" +#include "vhost.h" #define UDP_MAX_FRAMES 32 /* max # of frames to receive at once */ @@ -128,6 +129,8 @@ #define UDP_TIMEOUT_DEFAULT 30 /* s */ #define UDP_TIMEOUT_STREAM_DEFAULT 120 /* s */ +#define UDP_NREGIONS 5 + /* Maximum UDP data to be returned in ICMP messages */ #define ICMP4_MAX_DLEN 8 #define ICMP6_MAX_DLEN (IPV6_MIN_MTU \ @@ -448,6 +451,23 @@ static void udp_send_tap_icmp4(const struct ctx *c, tap_icmp4_send(c, saddr, eaddr, &msg, tap_omac, msglen); } +/** + * udp_register_memory_region() - Register the UDP specific buffers into a memory + * struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void udp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + UDP_NREGIONS) > N_VHOST_REGIONS) + die("udp: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + UDP_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_payload); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_eth_hdr); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_iov_recv); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_mh_recv); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(udp_meta); +} /** * udp_send_tap_icmp6() - Construct and send ICMPv6 to local peer diff --git a/udp.h b/udp.h index b50283e..ebf1a4e 100644 --- a/udp.h +++ b/udp.h @@ -11,6 +11,8 @@ #include "fwd.h" +union vhost_memory_u; + void udp_listen_sock_handler(const struct ctx *c, union epoll_ref ref, uint32_t events, const struct timespec *now); void udp_sock_handler(const struct ctx *c, union epoll_ref ref, @@ -21,6 +23,7 @@ int udp_tap_handler(const struct ctx *c, uint8_t pif, const struct timespec *now); int udp_init(struct ctx *c); void udp_update_l2_buf(const unsigned char *eth_d); +void udp_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); /** * struct udp_ctx - Execution context for UDP diff --git a/util.c b/util.c index ff1f19f..cafe0ec 100644 --- a/util.c +++ b/util.c @@ -33,10 +33,13 @@ #include "log.h" #include "pcap.h" #include "serialise.h" +#include "vhost.h" #ifdef HAS_GETRANDOM #include <sys/random.h> #endif +#define GENERAL_NREGIONS 2 + /* Zero-filled buffer to pad 802.3 frames, up to 60 (ETH_ZLEN) bytes */ uint8_t eth_pad[ETH_ZLEN] = { 0 }; @@ -453,6 +456,21 @@ int open_in_ns(const struct ctx *c, const char *path, int flags) return arg.fd; } +/** + * general_register_memory_region() - Register the general (non protocol specific) buffers + * into a memory struct so it can be shared with the kernel. + * @vhost_mem: The memory struct to register regions into + * @last_idx: The last index at which memory region has been placed + */ +void general_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx) { + if ((*last_idx + GENERAL_NREGIONS) > N_VHOST_REGIONS) + die("general: memory region overflow. maximum is %d, got %zu", + N_VHOST_REGIONS, *last_idx + GENERAL_NREGIONS); + + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(pkt_buf); + vhost_mem->mem.regions[(*last_idx)++] = VHOST_MEMORY_REGION(eth_pad); +} + /** * pidfile_write() - Write PID to file, if requested to do so, and close it * @fd: Open PID file descriptor, closed on exit, -1 to skip writing it diff --git a/util.h b/util.h index ec2d9ac..11f6877 100644 --- a/util.h +++ b/util.h @@ -145,6 +145,7 @@ int do_clone(int (*fn)(void *), char *stack_area, size_t stack_size, int flags, struct ctx; union sockaddr_inany; +union vhost_memory_u; int sock_l4(const struct ctx *c, enum epoll_type type, const union sockaddr_inany *sa, const char *ifname); @@ -170,6 +171,7 @@ int write_remainder(int fd, const struct iovec *iov, size_t iovcnt, int read_remainder(int fd, const struct iovec *iov, size_t cnt, size_t skip); bool snprintf_check(char *str, size_t size, const char *format, ...); long clamped_scale(long x, long y, long lo, long hi, long f); +void general_register_memory_regions(union vhost_memory_u *vhost_mem, size_t *last_idx); /** * af_name() - Return name of an address family diff --git a/vhost.c b/vhost.c new file mode 100644 index 0000000..a69c4b5 --- /dev/null +++ b/vhost.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Copyright Red Hat + * Author: Ammar Yasser <aerosound161(a)gmail.com> + * + * vhost.c - vhost-net (vhost-kernel) acceleration for pasta mode + */ + +#include <errno.h> +#include <fcntl.h> +#include <inttypes.h> +#include <stdint.h> +#include <string.h> +#include <unistd.h> +#include <sys/eventfd.h> +#include <sys/ioctl.h> + +#include "util.h" +#include "passt.h" +#include "vhost.h" +#include "epoll_ctl.h" +#include "udp.h" +#include "tcp_buf.h" + +struct vq_state vhost_vq_state[2]; + +struct vring_desc_pasta vring_desc[2][VHOST_NDESCS] + __attribute__((aligned(PAGE_SIZE))); + +struct vring_avail_pasta vring_avail_all[2] + __attribute__((aligned(PAGE_SIZE))); +struct vring_used_pasta vring_used_all[2] + __attribute__((aligned(PAGE_SIZE))); + +union vhost_memory_u vhost_memory = { + .mem = { + .nregions = N_VHOST_REGIONS, + }, +}; + +/** + * vhost_setup_net() - Open and negotiate features on /dev/vhost-net + * @c: Execution context; c->vhost.fd and c->vhost.features are set + * on success, and left untouched on failure + * + * Failure here means vhost-net isn't usable, not that pasta can't run: the + * caller falls back to plain tap operation unless acceleration was required. + * That's why nothing in here is fatal. + * + * Return: 0 on success, -1 if vhost-net is unavailable or unusable + */ +int vhost_setup_net(struct ctx *c) +{ + const uint64_t req_features = (1ULL << VIRTIO_F_VERSION_1) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t features; + int vhost_fd; + + vhost_fd = open("/dev/vhost-net", O_RDWR | O_NONBLOCK | O_CLOEXEC); + if (vhost_fd < 0) { + debug_perror("Couldn't open /dev/vhost-net"); + return -1; + } + + if (ioctl(vhost_fd, VHOST_SET_OWNER, NULL) < 0) { + debug_perror("VHOST_SET_OWNER ioctl failed"); + goto close_fd; + } + + if (ioctl(vhost_fd, VHOST_GET_FEATURES, &features) < 0) { + debug_perror("VHOST_GET_FEATURES ioctl failed"); + goto close_fd; + } + + if ((features & req_features) != req_features) { + debug("vhost-net is missing features: 0x%016" PRIx64, + req_features & ~features); + goto close_fd; + } + + features = req_features; + if (ioctl(vhost_fd, VHOST_SET_FEATURES, &features) < 0) { + debug_perror("VHOST_SET_FEATURES ioctl failed"); + goto close_fd; + } + + c->vhost.features = features; + c->vhost.fd = vhost_fd; + + return 0; + +close_fd: + close(vhost_fd); + + return -1; +} + +/** + * vhost_setup_eventfds() - Set up one queue's eventfds and ring size + * @c: Execution context; c->vhost.fd must already be set + * @queue_idx: Index of the queue (vring) to configure + * + */ +void vhost_setup_eventfds(struct ctx *c, int queue_idx) +{ + struct vhost_vring_file vring_file = { .index = queue_idx }; + struct vhost_vring_state state = { .index = queue_idx }; + union epoll_ref ref = { 0 }; + int vhost_fd = c->vhost.fd; + int rc; + + state.num = VHOST_NDESCS; + ref.queue = queue_idx; + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost call eventfd, queue %d", + queue_idx); + ref.fd = vring_file.fd; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_CALL, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_CALL ioctl failed, queue %d", + queue_idx); + + ref.type = EPOLL_TYPE_VHOST_CALL; + rc = epoll_add(c->epollfd, EPOLLIN, ref); + if (rc < 0) + die_perror("Failed to watch vhost call eventfd, queue %d", + queue_idx); + c->vhost.vq[queue_idx].call_fd = vring_file.fd; + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost error eventfd, queue %d", + queue_idx); + + rc = ioctl(vhost_fd, VHOST_SET_VRING_ERR, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_ERR ioctl failed, queue %d", + queue_idx); + + ref.type = EPOLL_TYPE_VHOST_ERROR; + ref.fd = vring_file.fd; + rc = epoll_add(c->epollfd, EPOLLIN, ref); + if (rc < 0) + die_perror("Failed to watch vhost error eventfd, queue %d", + queue_idx); + c->vhost.vq[queue_idx].err_fd = vring_file.fd; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_NUM, &state); + if (rc < 0) + die_perror("VHOST_SET_VRING_NUM ioctl failed, queue %d", + queue_idx); + + vring_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (vring_file.fd < 0) + die_perror("Failed to create vhost kick eventfd, queue %d", + queue_idx); + + rc = ioctl(vhost_fd, VHOST_SET_VRING_KICK, &vring_file); + if (rc < 0) + die_perror("VHOST_SET_VRING_KICK ioctl failed, queue %d", + queue_idx); + + c->vhost.vq[queue_idx].kick_fd = vring_file.fd; + + vhost_vq_state[queue_idx].num_free = VHOST_NDESCS; +} + +/** + * vhost_setup_memory_table() - Register the GPA/HVA translation table + * @c: Execution context; c->vhost.fd must already be set + * + * vhost-net reads the addresses we put in descriptors as guest physical + * addresses, and translates them through this table before touching the + * memory they refer to. pasta has no guest and no second address space: + * the addresses we put there are our own virtual addresses. Every region + * below therefore sets guest_phys_addr equal to userspace_addr, so that + * GPA == HVA and the translation is an identity mapping. + * + * Return: 0 on success, -1 on error with errno set + */ +int vhost_setup_memory_table(struct ctx *c) { + size_t region_idx = 0; + + /* general purpose buffers */ + general_register_memory_regions(&vhost_memory, &region_idx); + + /* tcp specific buffers */ + tcp_register_memory_regions(&vhost_memory, &region_idx); + + /* udp specific buffers */ + udp_register_memory_regions(&vhost_memory, &region_idx); + + vhost_memory.mem.nregions = region_idx; + + return ioctl(c->vhost.fd, VHOST_SET_MEM_TABLE, &vhost_memory.mem); +} + +/** + * vhost_set_vring() - Register a vring's addresses and bind its backend + * @c: Execution context; c->vhost.fd must already be set + * @queue_idx: Index of the queue (vring) to configure + * @tap_fd: Tap fd to bind as this queue's backend + */ +void vhost_set_vring(struct ctx *c, int queue_idx, int tap_fd) +{ + int vhost_fd = c->vhost.fd; + struct vhost_vring_addr addr = { + .index = queue_idx, + .desc_user_addr = (unsigned long)vring_desc[queue_idx], + .avail_user_addr = (unsigned long)&vring_avail_all[queue_idx], + .used_user_addr = (unsigned long)&vring_used_all[queue_idx], + .log_guest_addr = (unsigned long)&vring_used_all[queue_idx], + }; + struct vhost_vring_file file = { + .index = queue_idx, + .fd = tap_fd, + }; + unsigned int i; + int rc; + + rc = ioctl(vhost_fd, VHOST_SET_VRING_ADDR, &addr); + if (rc < 0) + die_perror("VHOST_SET_VRING_ADDR ioctl failed, queue %d", + queue_idx); + + if (queue_idx == VHOST_AVAIL_Q_IDX) { + for (i = 0; i < VHOST_NDESCS; ++i) { + vring_desc[VHOST_AVAIL_Q_IDX][i].addr = + (uintptr_t)pkt_buf + i * VHOST_DESC_BYTES; + vring_desc[VHOST_AVAIL_Q_IDX][i].len = VHOST_DESC_BYTES; + vring_desc[VHOST_AVAIL_Q_IDX][i].flags = + VRING_DESC_F_WRITE; + } + + for (i = 0; i < VHOST_NDESCS; ++i) + vring_avail_all[VHOST_AVAIL_Q_IDX].ring[i] = + htole16(i); + + vhost_rx_descriptor_handoff(c); + } + + if (queue_idx == VHOST_USED_Q_IDX) { + for (i = 0; i < (VHOST_NDESCS - 1); ++i) + vring_desc[VHOST_USED_Q_IDX][i].next = i + 1; + } + + rc = ioctl(vhost_fd, VHOST_NET_SET_BACKEND, &file); + if (rc < 0) + die_perror("VHOST_NET_SET_BACKEND ioctl failed, queue %d", + queue_idx); +} + +/** + * vhost_rx_descriptor_handoff() - Announce freed from-guest descriptors + * @c: Execution context + * + * Bumps avail.idx by the number of descriptors accumulated in + * vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free (from prior + * consume_one_rx_descriptor() calls), + * then resets the counter to zero. The kernel will see the new + * avail.idx and consume the freshly-available descriptors. + */ +void vhost_rx_descriptor_handoff(struct ctx *c) +{ + smp_wmb(); + + if (!vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free) + return; + + vring_avail_all[VHOST_AVAIL_Q_IDX].idx += + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free; + vhost_vq_state[VHOST_AVAIL_Q_IDX].num_free = 0; + vhost_kick(&vring_used_all[VHOST_AVAIL_Q_IDX], + c->vhost.vq[VHOST_AVAIL_Q_IDX].kick_fd); +} + +/** + * vhost_kick() - Notify the kernel that new descriptors are available + * @used: Used ring of the queue we're announcing on, checked to see + * whether the kernel's virtio thread is already reading + * descriptors and doesn't want to be notified + * @kick_fd: Kick eventfd of that same queue + */ +void vhost_kick(struct vring_used_pasta *used, int kick_fd) +{ + /* Ensure that the read of used->flags doesn't get reordered to be + * above the avail.idx update + */ + smp_mb(); + + if (!(used->flags & VRING_USED_F_NO_NOTIFY)) + eventfd_write(kick_fd, 1); +} diff --git a/vhost.h b/vhost.h new file mode 100644 index 0000000..bd555f9 --- /dev/null +++ b/vhost.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * Copyright Red Hat + * Author: Ammar Yasser <aerosound161(a)gmail.com> + * + * vhost.h - vhost-net (vhost-kernel) acceleration for pasta mode + */ + +#ifndef VHOST_H +#define VHOST_H + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> + +#include <linux/vhost.h> +#include <linux/vhost_types.h> +#include <linux/virtio_config.h> +#include <linux/virtio_net.h> +#include <linux/virtio_ring.h> + +#include "passt.h" + +#define VHOST_AVAIL_Q_IDX 0 +#define VHOST_USED_Q_IDX 1 + +/** + * struct vq_state - Per-virtqueue local descriptor tracking + * @num_free: Number of descriptors ready to be announced to the + * kernel via vhost_rx_descriptor_handoff() + * @last_used_idx: Number of used-ring entries consumed so far; + * lagging read cursor vs. vring_used->idx (the + * kernel's write cursor) + */ +extern struct vq_state { + uint16_t num_free; + uint16_t last_used_idx; + uint16_t next_free; +} vhost_vq_state[2]; + +struct vring_desc_pasta { + __virtio64 addr; + __virtio32 len; + __virtio16 flags; + __virtio16 next; +}; + +extern struct vring_desc_pasta vring_desc[2][VHOST_NDESCS]; + +struct vring_avail_pasta { + __virtio16 flags; + __virtio16 idx; + __virtio16 ring[VHOST_NDESCS]; +}; +static_assert(sizeof(struct vring_avail_pasta) == + offsetof(struct vring_avail, ring[VHOST_NDESCS]), + "vring_avail_pasta doesn't match the kernel avail ring layout"); +extern struct vring_avail_pasta vring_avail_all[2]; + +struct vring_used_pasta { + __virtio16 flags; + __virtio16 idx; + vring_used_elem_t ring[VHOST_NDESCS]; +}; +static_assert(sizeof(struct vring_used_pasta) == + offsetof(struct vring_used, ring[VHOST_NDESCS]), + "vring_used_pasta doesn't match the kernel used ring layout"); +extern struct vring_used_pasta vring_used_all[2]; + +#define N_VHOST_REGIONS 12 +union vhost_memory_u { + struct vhost_memory mem; + char buf[offsetof(struct vhost_memory, regions[N_VHOST_REGIONS])]; +}; +extern union vhost_memory_u vhost_memory; + +#define VHOST_MEMORY_REGION_PTR(addr, size) \ + (struct vhost_memory_region) { \ + .guest_phys_addr = (uintptr_t)addr, \ + .memory_size = size, \ + .userspace_addr = (uintptr_t)addr, \ + } +#define VHOST_MEMORY_REGION(buf) VHOST_MEMORY_REGION_PTR(&buf, sizeof(buf)) + +void vhost_set_vring(struct ctx *c, int queue_idx, int tap_fd); +int vhost_setup_memory_table(struct ctx *c); +int vhost_setup_net(struct ctx *c); +void vhost_setup_eventfds(struct ctx *c, int queue_idx); +void vhost_rx_descriptor_handoff(struct ctx *c); +void vhost_kick(struct vring_used_pasta *used, int kick_fd); + +#endif /* VHOST_H */ -- 2.39.5 (Apple Git-154)

1 0