In pasta mode, when we get data from sockets and write it as single
frames to the tap device, we batch receive operations considerably,
and then (conceptually) split the data in many smaller writes.
It looked like an obvious choice, but performance is actually better
if we receive data in many small frame-sized recvmsg()/recvmmsg().
The syscall overhead with the previous behaviour, observed by perf,
comes predominantly from write operations, but receiving data in
shorter chunks probably improves cache locality by a considerable
amount.
Signed-off-by: Stefano Brivio <sbrivio(a)redhat.com>
---
tcp.c | 36 ++++++++++++++++++++----------------
udp.c | 33 +++++++++++++++++----------------
2 files changed, 37 insertions(+), 32 deletions(-)
diff --git a/tcp.c b/tcp.c
index e4fac22..a3a9dfd 100644
--- a/tcp.c
+++ b/tcp.c
@@ -343,7 +343,9 @@
#define MAX_TAP_CONNS (128 * 1024)
#define MAX_SPLICE_CONNS (128 * 1024)
-#define TCP_TAP_FRAMES 256
+#define TCP_TAP_FRAMES_MEM 256
+#define TCP_TAP_FRAMES \
+ (c->mode == MODE_PASST ? TCP_TAP_FRAMES_MEM : 1)
#define MAX_PIPE_SIZE (2UL * 1024 * 1024)
@@ -609,7 +611,7 @@ static struct tcp4_l2_buf_t {
#else
} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif
-tcp4_l2_buf[TCP_TAP_FRAMES];
+tcp4_l2_buf[TCP_TAP_FRAMES_MEM];
static unsigned int tcp4_l2_buf_used;
static size_t tcp4_l2_buf_bytes;
@@ -640,21 +642,21 @@ struct tcp6_l2_buf_t {
#else
} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif
-tcp6_l2_buf[TCP_TAP_FRAMES];
+tcp6_l2_buf[TCP_TAP_FRAMES_MEM];
static unsigned int tcp6_l2_buf_used;
static size_t tcp6_l2_buf_bytes;
/* recvmsg()/sendmsg() data for tap */
static char tcp_buf_discard [MAX_WINDOW];
-static struct iovec iov_sock [TCP_TAP_FRAMES + 1];
+static struct iovec iov_sock [TCP_TAP_FRAMES_MEM + 1];
-static struct iovec tcp4_l2_iov_tap [TCP_TAP_FRAMES];
-static struct iovec tcp6_l2_iov_tap [TCP_TAP_FRAMES];
-static struct iovec tcp4_l2_flags_iov_tap [TCP_TAP_FRAMES];
-static struct iovec tcp6_l2_flags_iov_tap [TCP_TAP_FRAMES];
+static struct iovec tcp4_l2_iov_tap [TCP_TAP_FRAMES_MEM];
+static struct iovec tcp6_l2_iov_tap [TCP_TAP_FRAMES_MEM];
+static struct iovec tcp4_l2_flags_iov_tap [TCP_TAP_FRAMES_MEM];
+static struct iovec tcp6_l2_flags_iov_tap [TCP_TAP_FRAMES_MEM];
-static struct mmsghdr tcp_l2_mh_tap [TCP_TAP_FRAMES];
+static struct mmsghdr tcp_l2_mh_tap [TCP_TAP_FRAMES_MEM];
/* sendmsg() to socket */
static struct iovec tcp_tap_iov [UIO_MAXIOV];
@@ -688,7 +690,7 @@ static struct tcp4_l2_flags_buf_t {
#else
} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif
-tcp4_l2_flags_buf[TCP_TAP_FRAMES];
+tcp4_l2_flags_buf[TCP_TAP_FRAMES_MEM];
static int tcp4_l2_flags_buf_used;
@@ -717,7 +719,7 @@ static struct tcp6_l2_flags_buf_t {
#else
} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif
-tcp6_l2_flags_buf[TCP_TAP_FRAMES];
+tcp6_l2_flags_buf[TCP_TAP_FRAMES_MEM];
static int tcp6_l2_flags_buf_used;
@@ -916,7 +918,7 @@ void tcp_update_l2_buf(unsigned char *eth_d, unsigned char *eth_s,
{
int i;
- for (i = 0; i < TCP_TAP_FRAMES; i++) {
+ for (i = 0; i < TCP_TAP_FRAMES_MEM; i++) {
struct tcp4_l2_flags_buf_t *b4f = &tcp4_l2_flags_buf[i];
struct tcp6_l2_flags_buf_t *b6f = &tcp6_l2_flags_buf[i];
struct tcp4_l2_buf_t *b4 = &tcp4_l2_buf[i];
@@ -982,12 +984,13 @@ static void tcp_sock4_iov_init(void)
};
}
- for (i = 0, iov = tcp4_l2_iov_tap; i < TCP_TAP_FRAMES; i++, iov++) {
+ for (i = 0, iov = tcp4_l2_iov_tap; i < TCP_TAP_FRAMES_MEM; i++, iov++) {
iov->iov_base = &tcp4_l2_buf[i].vnet_len;
iov->iov_len = MSS_DEFAULT;
}
- for (i = 0, iov = tcp4_l2_flags_iov_tap; i < TCP_TAP_FRAMES; i++, iov++)
+ for (i = 0, iov = tcp4_l2_flags_iov_tap; i < TCP_TAP_FRAMES_MEM;
+ i++, iov++)
iov->iov_base = &tcp4_l2_flags_buf[i].vnet_len;
}
@@ -1015,12 +1018,13 @@ static void tcp_sock6_iov_init(void)
};
}
- for (i = 0, iov = tcp6_l2_iov_tap; i < TCP_TAP_FRAMES; i++, iov++) {
+ for (i = 0, iov = tcp6_l2_iov_tap; i < TCP_TAP_FRAMES_MEM; i++, iov++) {
iov->iov_base = &tcp6_l2_buf[i].vnet_len;
iov->iov_len = MSS_DEFAULT;
}
- for (i = 0, iov = tcp6_l2_flags_iov_tap; i < TCP_TAP_FRAMES; i++, iov++)
+ for (i = 0, iov = tcp6_l2_flags_iov_tap; i < TCP_TAP_FRAMES_MEM;
+ i++, iov++)
iov->iov_base = &tcp6_l2_flags_buf[i].vnet_len;
}
diff --git a/udp.c b/udp.c
index 8129a89..d4f3714 100644
--- a/udp.c
+++ b/udp.c
@@ -118,7 +118,8 @@
#define UDP_CONN_TIMEOUT 180 /* s, timeout for ephemeral or local bind */
#define UDP_SPLICE_FRAMES 128
-#define UDP_TAP_FRAMES 128
+#define UDP_TAP_FRAMES_MEM 128
+#define UDP_TAP_FRAMES (c->mode == MODE_PASST ? UDP_TAP_FRAMES_MEM : 1)
/**
* struct udp_tap_port - Port tracking based on tap-facing source port
@@ -204,7 +205,7 @@ static struct udp4_l2_buf_t {
uint8_t data[USHRT_MAX -
(sizeof(struct iphdr) + sizeof(struct udphdr))];
} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
-udp4_l2_buf[UDP_TAP_FRAMES];
+udp4_l2_buf[UDP_TAP_FRAMES_MEM];
/**
* udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections
@@ -234,23 +235,23 @@ struct udp6_l2_buf_t {
#else
} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif
-udp6_l2_buf[UDP_TAP_FRAMES];
+udp6_l2_buf[UDP_TAP_FRAMES_MEM];
static struct sockaddr_storage udp_splice_namebuf;
static uint8_t udp_splice_buf[UDP_SPLICE_FRAMES][USHRT_MAX];
/* recvmmsg()/sendmmsg() data for tap */
-static struct iovec udp4_l2_iov_sock [UDP_TAP_FRAMES];
-static struct iovec udp6_l2_iov_sock [UDP_TAP_FRAMES];
+static struct iovec udp4_l2_iov_sock [UDP_TAP_FRAMES_MEM];
+static struct iovec udp6_l2_iov_sock [UDP_TAP_FRAMES_MEM];
-static struct iovec udp4_l2_iov_tap [UDP_TAP_FRAMES];
-static struct iovec udp6_l2_iov_tap [UDP_TAP_FRAMES];
+static struct iovec udp4_l2_iov_tap [UDP_TAP_FRAMES_MEM];
+static struct iovec udp6_l2_iov_tap [UDP_TAP_FRAMES_MEM];
-static struct mmsghdr udp4_l2_mh_sock [UDP_TAP_FRAMES];
-static struct mmsghdr udp6_l2_mh_sock [UDP_TAP_FRAMES];
+static struct mmsghdr udp4_l2_mh_sock [UDP_TAP_FRAMES_MEM];
+static struct mmsghdr udp6_l2_mh_sock [UDP_TAP_FRAMES_MEM];
-static struct mmsghdr udp4_l2_mh_tap [UDP_TAP_FRAMES];
-static struct mmsghdr udp6_l2_mh_tap [UDP_TAP_FRAMES];
+static struct mmsghdr udp4_l2_mh_tap [UDP_TAP_FRAMES_MEM];
+static struct mmsghdr udp6_l2_mh_tap [UDP_TAP_FRAMES_MEM];
/* recvmmsg()/sendmmsg() data for "spliced" connections */
static struct iovec udp_splice_iov_recv [UDP_SPLICE_FRAMES];
@@ -310,7 +311,7 @@ void udp_update_l2_buf(unsigned char *eth_d, unsigned char *eth_s,
{
int i;
- for (i = 0; i < UDP_TAP_FRAMES; i++) {
+ for (i = 0; i < UDP_TAP_FRAMES_MEM; i++) {
struct udp4_l2_buf_t *b4 = &udp4_l2_buf[i];
struct udp6_l2_buf_t *b6 = &udp6_l2_buf[i];
@@ -354,7 +355,7 @@ static void udp_sock4_iov_init(void)
};
}
- for (i = 0, h = udp4_l2_mh_sock; i < UDP_TAP_FRAMES; i++, h++) {
+ for (i = 0, h = udp4_l2_mh_sock; i < UDP_TAP_FRAMES_MEM; i++, h++) {
struct msghdr *mh = &h->msg_hdr;
mh->msg_name = &udp4_l2_buf[i].s_in;
@@ -366,7 +367,7 @@ static void udp_sock4_iov_init(void)
mh->msg_iovlen = 1;
}
- for (i = 0, h = udp4_l2_mh_tap; i < UDP_TAP_FRAMES; i++, h++) {
+ for (i = 0, h = udp4_l2_mh_tap; i < UDP_TAP_FRAMES_MEM; i++, h++) {
struct msghdr *mh = &h->msg_hdr;
udp4_l2_iov_tap[i].iov_base = &udp4_l2_buf[i].vnet_len;
@@ -394,7 +395,7 @@ static void udp_sock6_iov_init(void)
};
}
- for (i = 0, h = udp6_l2_mh_sock; i < UDP_TAP_FRAMES; i++, h++) {
+ for (i = 0, h = udp6_l2_mh_sock; i < UDP_TAP_FRAMES_MEM; i++, h++) {
struct msghdr *mh = &h->msg_hdr;
mh->msg_name = &udp6_l2_buf[i].s_in6;
@@ -406,7 +407,7 @@ static void udp_sock6_iov_init(void)
mh->msg_iovlen = 1;
}
- for (i = 0, h = udp6_l2_mh_tap; i < UDP_TAP_FRAMES; i++, h++) {
+ for (i = 0, h = udp6_l2_mh_tap; i < UDP_TAP_FRAMES_MEM; i++, h++) {
struct msghdr *mh = &h->msg_hdr;
udp6_l2_iov_tap[i].iov_base = &udp6_l2_buf[i].vnet_len;
--
2.34.1