nl_link() performs a number of functions: it can bring links up, set MAC
address and MTU and also retrieve the existing MAC.  This makes for a small
number of lines of code, but high conceptual complexity: it's quite hard
to follow what's going on both in nl_link() itself and it's also not very
obvious which function its callers are intending to use.

Clarify this, by splitting nl_link() into nl_link_up(), nl_link_set_mac(),
and nl_link_get_mac().  The first brings up a link, optionally setting the
MTU, the others get or set the MAC address.

This fixes an arguable bug in pasta_ns_conf(): it looks as though that was
intended to retrieve the guest MAC whether or not c->pasta_conf_ns is set.
However, it only actually does so in the !c->pasta_conf_ns case: the fact
that we set up==1 means we would only ever set, never get, the MAC in the
nl_link() call in the other path.  We get away with this because the MAC
will quickly be discovered once we receive packets on the tap interface.
Still, it's neater to always get the MAC address here.

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 conf.c    |   4 +-
 netlink.c | 143 +++++++++++++++++++++++++++++++-----------------------
 netlink.h |   4 +-
 pasta.c   |  12 +++--
 4 files changed, 96 insertions(+), 67 deletions(-)

diff --git a/conf.c b/conf.c
index 78eaf2d..2ff9e2a 100644
--- a/conf.c
+++ b/conf.c
@@ -670,7 +670,7 @@ static unsigned int conf_ip4(unsigned int ifi,
 	memcpy(&ip4->addr_seen, &ip4->addr, sizeof(ip4->addr_seen));
 
 	if (MAC_IS_ZERO(mac))
-		nl_link(0, ifi, mac, 0, 0);
+		nl_link_get_mac(0, ifi, mac);
 
 	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr) ||
 	    MAC_IS_ZERO(mac))
@@ -711,7 +711,7 @@ static unsigned int conf_ip6(unsigned int ifi,
 	memcpy(&ip6->addr_ll_seen, &ip6->addr_ll, sizeof(ip6->addr_ll));
 
 	if (MAC_IS_ZERO(mac))
-		nl_link(0, ifi, mac, 0, 0);
+		nl_link_get_mac(0, ifi, mac);
 
 	if (IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) ||
 	    IN6_IS_ADDR_UNSPECIFIED(&ip6->addr_ll) ||
diff --git a/netlink.c b/netlink.c
index e15e23f..4b1f75e 100644
--- a/netlink.c
+++ b/netlink.c
@@ -486,83 +486,44 @@ next:
 }
 
 /**
- * nl_link() - Get/set link attributes
+ * nl_link_get_mac() - Get link MAC address
  * @ns:		Use netlink socket in namespace
  * @ifi:	Interface index
- * @mac:	MAC address to fill, if passed as zero, to set otherwise
- * @up:		If set, bring up the link
- * @mtu:	If non-zero, set interface MTU
+ * @mac:	Fill with current MAC address
  */
-void nl_link(int ns, unsigned int ifi, void *mac, int up, int mtu)
+void nl_link_get_mac(int ns, unsigned int ifi, void *mac)
 {
-	int change = !MAC_IS_ZERO(mac) || up || mtu;
 	struct req_t {
 		struct nlmsghdr nlh;
 		struct ifinfomsg ifm;
-		struct rtattr rta;
-		union {
-			unsigned char mac[ETH_ALEN];
-			struct {
-				unsigned int mtu;
-			} mtu;
-		} set;
 	} req = {
-		.nlh.nlmsg_type   = change ? RTM_NEWLINK : RTM_GETLINK,
-		.nlh.nlmsg_len    = NLMSG_LENGTH(sizeof(struct ifinfomsg)),
-		.nlh.nlmsg_flags  = NLM_F_REQUEST | (change ? NLM_F_ACK : 0),
+		.nlh.nlmsg_type	  = RTM_GETLINK,
+		.nlh.nlmsg_len	  = sizeof(req),
+		.nlh.nlmsg_flags  = NLM_F_REQUEST | NLM_F_ACK,
 		.nlh.nlmsg_seq	  = nl_seq++,
 		.ifm.ifi_family	  = AF_UNSPEC,
 		.ifm.ifi_index	  = ifi,
-		.ifm.ifi_flags	  = up ? IFF_UP : 0,
-		.ifm.ifi_change	  = up ? IFF_UP : 0,
 	};
-	struct ifinfomsg *ifm;
 	struct nlmsghdr *nh;
-	struct rtattr *rta;
 	char buf[NLBUFSIZ];
 	ssize_t n;
-	size_t na;
-
-	if (!MAC_IS_ZERO(mac)) {
-		req.nlh.nlmsg_len = sizeof(req);
-		memcpy(req.set.mac, mac, ETH_ALEN);
-		req.rta.rta_type = IFLA_ADDRESS;
-		req.rta.rta_len = RTA_LENGTH(ETH_ALEN);
-		if (nl_req(ns, buf, &req, req.nlh.nlmsg_len) < 0)
-			return;
-
-		up = 0;
-	}
-
-	if (mtu) {
-		req.nlh.nlmsg_len = offsetof(struct req_t, set.mtu)
-			+ sizeof(req.set.mtu);
-		req.set.mtu.mtu = mtu;
-		req.rta.rta_type = IFLA_MTU;
-		req.rta.rta_len = RTA_LENGTH(sizeof(unsigned int));
-		if (nl_req(ns, buf, &req, req.nlh.nlmsg_len) < 0)
-			return;
-
-		up = 0;
-	}
-
-	if (up && nl_req(ns, buf, &req, req.nlh.nlmsg_len) < 0)
-		return;
-
-	if (change)
-		return;
 
-	if ((n = nl_req(ns, buf, &req, req.nlh.nlmsg_len)) < 0)
+	n = nl_req(ns, buf, &req, sizeof(req));
+	if (n < 0)
 		return;
+	
+	for (nh = (struct nlmsghdr *)buf;
+	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
+	     nh = NLMSG_NEXT(nh, n)) {
+		struct ifinfomsg *ifm = (struct ifinfomsg *)NLMSG_DATA(nh);
+		struct rtattr *rta;
+		size_t na;
 
-	nh = (struct nlmsghdr *)buf;
-	for ( ; NLMSG_OK(nh, n); nh = NLMSG_NEXT(nh, n)) {
 		if (nh->nlmsg_type != RTM_NEWLINK)
-			goto next;
-
-		ifm = (struct ifinfomsg *)NLMSG_DATA(nh);
+			continue;
 
-		for (rta = IFLA_RTA(ifm), na = RTM_PAYLOAD(nh); RTA_OK(rta, na);
+		for (rta = IFLA_RTA(ifm), na = RTM_PAYLOAD(nh);
+		     RTA_OK(rta, na);
 		     rta = RTA_NEXT(rta, na)) {
 			if (rta->rta_type != IFLA_ADDRESS)
 				continue;
@@ -570,8 +531,70 @@ void nl_link(int ns, unsigned int ifi, void *mac, int up, int mtu)
 			memcpy(mac, RTA_DATA(rta), ETH_ALEN);
 			break;
 		}
-next:
-		if (nh->nlmsg_type == NLMSG_DONE)
-			break;
 	}
 }
+
+/**
+ * nl_link_set_mac() - Set link MAC address
+ * @ns:		Use netlink socket in namespace
+ * @ifi:	Interface index
+ * @mac:	MAC address to set
+ */
+void nl_link_set_mac(int ns, unsigned int ifi, void *mac)
+{
+	struct req_t {
+		struct nlmsghdr nlh;
+		struct ifinfomsg ifm;
+		struct rtattr rta;
+		unsigned char mac[ETH_ALEN];
+	} req = {
+		.nlh.nlmsg_type	  = RTM_NEWLINK,
+		.nlh.nlmsg_len	  = sizeof(req),
+		.nlh.nlmsg_flags  = NLM_F_REQUEST | NLM_F_ACK,
+		.nlh.nlmsg_seq	  = nl_seq++,
+		.ifm.ifi_family	  = AF_UNSPEC,
+		.ifm.ifi_index	  = ifi,
+		.rta.rta_type	  = IFLA_ADDRESS,
+		.rta.rta_len	  = RTA_LENGTH(ETH_ALEN),
+	};
+	char buf[NLBUFSIZ];
+
+	memcpy(req.mac, mac, ETH_ALEN);
+
+	nl_req(ns, buf, &req, sizeof(req));
+}
+
+/**
+ * nl_link_up() - Bring link up
+ * @ns:		Use netlink socket in namespace
+ * @ifi:	Interface index
+ * @mtu:	If non-zero, set interface MTU
+ */
+void nl_link_up(int ns, unsigned int ifi, int mtu)
+{
+	struct req_t {
+		struct nlmsghdr nlh;
+		struct ifinfomsg ifm;
+		struct rtattr rta;
+		unsigned int mtu;
+	} req = {
+		.nlh.nlmsg_type   = RTM_NEWLINK,
+		.nlh.nlmsg_len    = sizeof(req),
+		.nlh.nlmsg_flags  = NLM_F_REQUEST | NLM_F_ACK,
+		.nlh.nlmsg_seq	  = nl_seq++,
+		.ifm.ifi_family	  = AF_UNSPEC,
+		.ifm.ifi_index	  = ifi,
+		.ifm.ifi_flags	  = IFF_UP,
+		.ifm.ifi_change	  = IFF_UP,
+		.rta.rta_type	  = IFLA_MTU,
+		.rta.rta_len	  = RTA_LENGTH(sizeof(unsigned int)),
+		.mtu		  = mtu,
+	};
+	char buf[NLBUFSIZ];
+
+	if (!mtu)
+		/* Shorten request to drop MTU attribute */
+		req.nlh.nlmsg_len = offsetof(struct req_t, rta);
+
+	nl_req(ns, buf, &req, req.nlh.nlmsg_len);
+}
diff --git a/netlink.h b/netlink.h
index cd0e666..980ac44 100644
--- a/netlink.h
+++ b/netlink.h
@@ -18,6 +18,8 @@ void nl_route(enum nl_op op, unsigned int ifi, unsigned int ifi_ns,
 	      sa_family_t af, void *gw);
 void nl_addr(enum nl_op op, unsigned int ifi, unsigned int ifi_ns,
 	     sa_family_t af, void *addr, int *prefix_len, void *addr_l);
-void nl_link(int ns, unsigned int ifi, void *mac, int up, int mtu);
+void nl_link_get_mac(int ns, unsigned int ifi, void *mac);
+void nl_link_set_mac(int ns, unsigned int ifi, void *mac);
+void nl_link_up(int ns, unsigned int ifi, int mtu);
 
 #endif /* NETLINK_H */
diff --git a/pasta.c b/pasta.c
index 8c85546..3b5537d 100644
--- a/pasta.c
+++ b/pasta.c
@@ -272,13 +272,19 @@ void pasta_start_ns(struct ctx *c, uid_t uid, gid_t gid,
  */
 void pasta_ns_conf(struct ctx *c)
 {
-	nl_link(1, 1 /* lo */, MAC_ZERO, 1, 0);
+	nl_link_up(1, 1 /* lo */, 0);
+
+	/* Get or set guest MAC */
+	if (MAC_IS_ZERO(c->mac_guest))
+		nl_link_get_mac(1, c->pasta_ifi, c->mac_guest);
+	else
+		nl_link_set_mac(1, c->pasta_ifi, c->mac_guest);
 
 	if (c->pasta_conf_ns) {
 		enum nl_op op_routes = c->no_copy_routes ? NL_SET : NL_DUP;
 		enum nl_op op_addrs =  c->no_copy_addrs  ? NL_SET : NL_DUP;
 
-		nl_link(1, c->pasta_ifi, c->mac_guest, 1, c->mtu);
+		nl_link_up(1, c->pasta_ifi, c->mtu);
 
 		if (c->ifi4) {
 			nl_addr(op_addrs, c->ifi4, c->pasta_ifi, AF_INET,
@@ -294,8 +300,6 @@ void pasta_ns_conf(struct ctx *c)
 			nl_route(op_routes, c->ifi6, c->pasta_ifi, AF_INET6,
 				 &c->ip6.gw);
 		}
-	} else {
-		nl_link(1, c->pasta_ifi, c->mac_guest, 0, 0);
 	}
 
 	proto_update_l2_buf(c->mac_guest, NULL, NULL);
-- 
2.41.0

nl_addr() can perform three quite different operations based on the 'op'
parameter, each of which uses a different subset of the parameters.  Split
them up into a function for each operation.  This does use more lines of
code, but the overlap wasn't that great, and the separated logic is much
easier to follow.

It's also clearer in the callers what we expect the netlink operations to
do, and what information it uses.

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 conf.c    |  12 ++-
 netlink.c | 232 ++++++++++++++++++++++++++++++++----------------------
 netlink.h |   6 +-
 pasta.c   |  17 ++--
 4 files changed, 159 insertions(+), 108 deletions(-)

diff --git a/conf.c b/conf.c
index 2ff9e2a..2057028 100644
--- a/conf.c
+++ b/conf.c
@@ -650,10 +650,8 @@ static unsigned int conf_ip4(unsigned int ifi,
 	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->gw))
 		nl_route(NL_GET, ifi, 0, AF_INET, &ip4->gw);
 
-	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr)) {
-		nl_addr(NL_GET, ifi, 0, AF_INET,
-			&ip4->addr, &ip4->prefix_len, NULL);
-	}
+	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr))
+		nl_addr_get(ifi, AF_INET, &ip4->addr, &ip4->prefix_len, NULL);
 
 	if (!ip4->prefix_len) {
 		in_addr_t addr = ntohl(ip4->addr.s_addr);
@@ -703,9 +701,9 @@ static unsigned int conf_ip6(unsigned int ifi,
 	if (IN6_IS_ADDR_UNSPECIFIED(&ip6->gw))
 		nl_route(NL_GET, ifi, 0, AF_INET6, &ip6->gw);
 
-	nl_addr(NL_GET, ifi, 0, AF_INET6,
-		IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) ? &ip6->addr : NULL,
-		&prefix_len, &ip6->addr_ll);
+	nl_addr_get(ifi, AF_INET6,
+		    IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) ? &ip6->addr : NULL,
+		    &prefix_len, &ip6->addr_ll);
 
 	memcpy(&ip6->addr_seen, &ip6->addr, sizeof(ip6->addr));
 	memcpy(&ip6->addr_ll_seen, &ip6->addr_ll, sizeof(ip6->addr_ll));
diff --git a/netlink.c b/netlink.c
index 4b1f75e..269d738 100644
--- a/netlink.c
+++ b/netlink.c
@@ -334,17 +334,76 @@ next:
 }
 
 /**
- * nl_addr() - Get/set/copy IP addresses for given interface and address family
- * @op:		Requested operation
+ * nl_addr_get() - Get IP address for given interface and address family
  * @ifi:	Interface index in outer network namespace
- * @ifi_ns:	Interface index in target namespace for NL_SET, NL_DUP
  * @af:		Address family
- * @addr:	Global address to fill on NL_GET, to set on NL_SET
- * @prefix_len:	Mask or prefix length, set or fetched (for IPv4)
- * @addr_l:	Link-scoped address to fill on NL_GET
+ * @addr:	Global address to fill
+ * @prefix_len:	Mask or prefix length, to fill (for IPv4)
+ * @addr_l:	Link-scoped address to fill (for IPv6)
+ */
+void nl_addr_get(unsigned int ifi, sa_family_t af, void *addr,
+		 int *prefix_len, void *addr_l)
+{
+	struct req_t {
+		struct nlmsghdr nlh;
+		struct ifaddrmsg ifa;
+	} req = {
+		.nlh.nlmsg_type    = RTM_GETADDR,
+		.nlh.nlmsg_flags   = NLM_F_REQUEST | NLM_F_ACK | NLM_F_DUMP,
+		.nlh.nlmsg_len     = sizeof(req),
+		.nlh.nlmsg_seq     = nl_seq++,
+
+		.ifa.ifa_family    = af,
+		.ifa.ifa_index     = ifi,
+	};
+	struct nlmsghdr *nh;
+	char buf[NLBUFSIZ];
+	ssize_t n;
+
+	if ((n = nl_req(0, buf, &req, req.nlh.nlmsg_len)) < 0)
+		return;
+
+	for (nh = (struct nlmsghdr *)buf;
+	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
+	     nh = NLMSG_NEXT(nh, n)) {
+		struct ifaddrmsg *ifa = (struct ifaddrmsg *)NLMSG_DATA(nh);
+		struct rtattr *rta;
+		size_t na;
+
+		if (nh->nlmsg_type != RTM_NEWADDR)
+			continue;
+
+		if (ifa->ifa_index != ifi)
+			continue;
+
+		for (rta = IFA_RTA(ifa), na = RTM_PAYLOAD(nh); RTA_OK(rta, na);
+		     rta = RTA_NEXT(rta, na)) {
+			if (rta->rta_type != IFA_ADDRESS)
+				continue;
+
+			if (af == AF_INET) {
+				memcpy(addr, RTA_DATA(rta), RTA_PAYLOAD(rta));
+				*prefix_len = ifa->ifa_prefixlen;
+			} else if (af == AF_INET6 && addr &&
+				   ifa->ifa_scope == RT_SCOPE_UNIVERSE) {
+				memcpy(addr, RTA_DATA(rta), RTA_PAYLOAD(rta));
+			}
+
+			if (addr_l &&
+			    af == AF_INET6 && ifa->ifa_scope == RT_SCOPE_LINK)
+				memcpy(addr_l, RTA_DATA(rta), RTA_PAYLOAD(rta));
+		}
+	}
+}
+
+/**
+ * nl_add_set() - Set IP addresses for given interface and address family
+ * @ifi:	Interface index
+ * @af:		Address family
+ * @addr:	Global address to set
+ * @prefix_len:	Mask or prefix length to set
  */
-void nl_addr(enum nl_op op, unsigned int ifi, unsigned int ifi_ns,
-	     sa_family_t af, void *addr, int *prefix_len, void *addr_l)
+void nl_addr_set(unsigned int ifi, sa_family_t af, void *addr, int prefix_len)
 {
 	struct req_t {
 		struct nlmsghdr nlh;
@@ -364,125 +423,112 @@ void nl_addr(enum nl_op op, unsigned int ifi, unsigned int
ifi_ns,
 			} a6;
 		} set;
 	} req = {
-		.nlh.nlmsg_type    = op == NL_SET ? RTM_NEWADDR : RTM_GETADDR,
-		.nlh.nlmsg_flags   = NLM_F_REQUEST,
+		.nlh.nlmsg_type    = RTM_NEWADDR,
+		.nlh.nlmsg_flags   = NLM_F_REQUEST | NLM_F_ACK |
+				     NLM_F_CREATE | NLM_F_EXCL,
 		.nlh.nlmsg_len     = NLMSG_LENGTH(sizeof(struct ifaddrmsg)),
 		.nlh.nlmsg_seq     = nl_seq++,
 
 		.ifa.ifa_family    = af,
-		.ifa.ifa_index     = op == NL_SET ? ifi_ns : ifi,
-		.ifa.ifa_prefixlen = op == NL_SET ? *prefix_len : 0,
+		.ifa.ifa_index     = ifi,
+		.ifa.ifa_prefixlen = prefix_len,
+		.ifa.ifa_scope	   = RT_SCOPE_UNIVERSE,
 	};
-	ssize_t n, nlmsgs_size;
-	struct ifaddrmsg *ifa;
-	struct nlmsghdr *nh;
-	struct rtattr *rta;
 	char buf[NLBUFSIZ];
-	size_t na;
 
-	if (op == NL_SET) {
-		if (af == AF_INET6) {
-			size_t rta_len = RTA_LENGTH(sizeof(req.set.a6.l));
+	if (af == AF_INET6) {
+		size_t rta_len = RTA_LENGTH(sizeof(req.set.a6.l));
 
-			/* By default, strictly speaking, it's duplicated */
-			req.ifa.ifa_flags = IFA_F_NODAD;
+		/* By default, strictly speaking, it's duplicated */
+		req.ifa.ifa_flags = IFA_F_NODAD;
 
-			req.nlh.nlmsg_len = offsetof(struct req_t, set.a6)
-				+ sizeof(req.set.a6);
+		req.nlh.nlmsg_len = offsetof(struct req_t, set.a6)
+			+ sizeof(req.set.a6);
 
-			memcpy(&req.set.a6.l, addr, sizeof(req.set.a6.l));
-			req.set.a6.rta_l.rta_len = rta_len;
-			req.set.a4.rta_l.rta_type = IFA_LOCAL;
-			memcpy(&req.set.a6.a, addr, sizeof(req.set.a6.a));
-			req.set.a6.rta_a.rta_len = rta_len;
-			req.set.a6.rta_a.rta_type = IFA_ADDRESS;
-		} else {
-			size_t rta_len = RTA_LENGTH(sizeof(req.set.a4.l));
-
-			req.nlh.nlmsg_len = offsetof(struct req_t, set.a4)
-				+ sizeof(req.set.a4);
+		memcpy(&req.set.a6.l, addr, sizeof(req.set.a6.l));
+		req.set.a6.rta_l.rta_len = rta_len;
+		req.set.a4.rta_l.rta_type = IFA_LOCAL;
+		memcpy(&req.set.a6.a, addr, sizeof(req.set.a6.a));
+		req.set.a6.rta_a.rta_len = rta_len;
+		req.set.a6.rta_a.rta_type = IFA_ADDRESS;
+	} else {
+		size_t rta_len = RTA_LENGTH(sizeof(req.set.a4.l));
 
-			req.set.a4.l = req.set.a4.a = *(uint32_t *)addr;
-			req.set.a4.rta_l.rta_len = rta_len;
-			req.set.a4.rta_l.rta_type = IFA_LOCAL;
-			req.set.a4.rta_a.rta_len = rta_len;
-			req.set.a4.rta_a.rta_type = IFA_ADDRESS;
-		}
+		req.nlh.nlmsg_len = offsetof(struct req_t, set.a4)
+			+ sizeof(req.set.a4);
 
-		req.ifa.ifa_scope = RT_SCOPE_UNIVERSE;
-		req.nlh.nlmsg_flags |= NLM_F_CREATE | NLM_F_ACK | NLM_F_EXCL;
-	} else {
-		req.nlh.nlmsg_flags |= NLM_F_DUMP;
+		req.set.a4.l = req.set.a4.a = *(uint32_t *)addr;
+		req.set.a4.rta_l.rta_len = rta_len;
+		req.set.a4.rta_l.rta_type = IFA_LOCAL;
+		req.set.a4.rta_a.rta_len = rta_len;
+		req.set.a4.rta_a.rta_type = IFA_ADDRESS;
 	}
 
-	if ((n = nl_req(op == NL_SET, buf, &req, req.nlh.nlmsg_len)) < 0)
-		return;
+	nl_req(1, buf, &req, req.nlh.nlmsg_len);
+}
 
-	if (op == NL_SET)
+/**
+ * nl_addr_dup() - Copy IP addresses for given interface and address family
+ * @ifi:	Interface index in outer network namespace
+ * @ifi_ns:	Interface index in target namespace
+ * @af:		Address family
+ */
+void nl_addr_dup(unsigned int ifi, unsigned int ifi_ns, sa_family_t af)
+{
+	struct req_t {
+		struct nlmsghdr nlh;
+		struct ifaddrmsg ifa;
+	} req = {
+		.nlh.nlmsg_type    = RTM_GETADDR,
+		.nlh.nlmsg_flags   = NLM_F_REQUEST | NLM_F_DUMP,
+		.nlh.nlmsg_len     = sizeof(req),
+		.nlh.nlmsg_seq     = nl_seq++,
+
+		.ifa.ifa_family    = af,
+		.ifa.ifa_index     = ifi,
+		.ifa.ifa_prefixlen = 0,
+	};
+	char buf[NLBUFSIZ], resp[NLBUFSIZ];
+	ssize_t n, nlmsgs_size;
+	struct nlmsghdr *nh;
+
+	if ((n = nl_req(0, buf, &req, sizeof(req))) < 0)
 		return;
 
-	nh = (struct nlmsghdr *)buf;
 	nlmsgs_size = n;
 
-	for ( ; NLMSG_OK(nh, n); nh = NLMSG_NEXT(nh, n)) {
+	for (nh = (struct nlmsghdr *)buf;
+	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
+	     nh = NLMSG_NEXT(nh, n)) {
+		struct ifaddrmsg *ifa;
+		struct rtattr *rta;
+		size_t na;
+
 		if (nh->nlmsg_type != RTM_NEWADDR)
-			goto next;
+			continue;
 
-		if (op == NL_DUP) {
-			nh->nlmsg_seq = nl_seq++;
-			nh->nlmsg_pid = 0;
-			nh->nlmsg_flags &= ~NLM_F_DUMP_FILTERED;
-			nh->nlmsg_flags |= NLM_F_REQUEST | NLM_F_ACK |
-					   NLM_F_CREATE;
-		}
+		nh->nlmsg_seq = nl_seq++;
+		nh->nlmsg_pid = 0;
+		nh->nlmsg_flags &= ~NLM_F_DUMP_FILTERED;
+		nh->nlmsg_flags |= NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE;
 
 		ifa = (struct ifaddrmsg *)NLMSG_DATA(nh);
 
-		if (op == NL_DUP && (ifa->ifa_scope == RT_SCOPE_LINK ||
-				     ifa->ifa_index != ifi)) {
+		if (ifa->ifa_scope == RT_SCOPE_LINK || ifa->ifa_index != ifi) {
 			ifa->ifa_family = AF_UNSPEC;
-			goto next;
+			continue;
 		}
 
-		if (ifa->ifa_index != ifi)
-			goto next;
-
-		if (op == NL_DUP)
-			ifa->ifa_index = ifi_ns;
+		ifa->ifa_index = ifi_ns;
 
 		for (rta = IFA_RTA(ifa), na = RTM_PAYLOAD(nh); RTA_OK(rta, na);
 		     rta = RTA_NEXT(rta, na)) {
-			if (op == NL_DUP && rta->rta_type == IFA_LABEL)
+			if (rta->rta_type == IFA_LABEL)
 				rta->rta_type = IFA_UNSPEC;
-
-			if (op == NL_DUP || rta->rta_type != IFA_ADDRESS)
-				continue;
-
-			if (af == AF_INET && addr && !*(uint32_t *)addr) {
-				memcpy(addr, RTA_DATA(rta), RTA_PAYLOAD(rta));
-				*prefix_len = ifa->ifa_prefixlen;
-			} else if (af == AF_INET6 && addr &&
-				 ifa->ifa_scope == RT_SCOPE_UNIVERSE &&
-				 IN6_IS_ADDR_UNSPECIFIED(addr)) {
-				memcpy(addr, RTA_DATA(rta), RTA_PAYLOAD(rta));
-			}
-
-			if (addr_l &&
-			    af == AF_INET6 && ifa->ifa_scope == RT_SCOPE_LINK &&
-			    IN6_IS_ADDR_UNSPECIFIED(addr_l))
-				memcpy(addr_l, RTA_DATA(rta), RTA_PAYLOAD(rta));
 		}
-next:
-		if (nh->nlmsg_type == NLMSG_DONE)
-			break;
 	}
 
-	if (op == NL_DUP) {
-		char resp[NLBUFSIZ];
-
-		nh = (struct nlmsghdr *)buf;
-		nl_req(1, resp, nh, nlmsgs_size);
-	}
+	nl_req(1, resp, buf, nlmsgs_size);
 }
 
 /**
diff --git a/netlink.h b/netlink.h
index 980ac44..5ac972d 100644
--- a/netlink.h
+++ b/netlink.h
@@ -16,8 +16,10 @@ void nl_sock_init(const struct ctx *c, bool ns);
 unsigned int nl_get_ext_if(sa_family_t af);
 void nl_route(enum nl_op op, unsigned int ifi, unsigned int ifi_ns,
 	      sa_family_t af, void *gw);
-void nl_addr(enum nl_op op, unsigned int ifi, unsigned int ifi_ns,
-	     sa_family_t af, void *addr, int *prefix_len, void *addr_l);
+void nl_addr_get(unsigned int ifi, sa_family_t af, void *addr,
+		 int *prefix_len, void *addr_l);
+void nl_addr_set(unsigned int ifi, sa_family_t af, void *addr, int prefix_len);
+void nl_addr_dup(unsigned int ifi, unsigned int ifi_ns, sa_family_t af);
 void nl_link_get_mac(int ns, unsigned int ifi, void *mac);
 void nl_link_set_mac(int ns, unsigned int ifi, void *mac);
 void nl_link_up(int ns, unsigned int ifi, int mtu);
diff --git a/pasta.c b/pasta.c
index 3b5537d..1a8f09c 100644
--- a/pasta.c
+++ b/pasta.c
@@ -282,21 +282,26 @@ void pasta_ns_conf(struct ctx *c)
 
 	if (c->pasta_conf_ns) {
 		enum nl_op op_routes = c->no_copy_routes ? NL_SET : NL_DUP;
-		enum nl_op op_addrs =  c->no_copy_addrs  ? NL_SET : NL_DUP;
 
 		nl_link_up(1, c->pasta_ifi, c->mtu);
 
 		if (c->ifi4) {
-			nl_addr(op_addrs, c->ifi4, c->pasta_ifi, AF_INET,
-				&c->ip4.addr, &c->ip4.prefix_len, NULL);
+			if (c->no_copy_addrs)
+				nl_addr_set(c->pasta_ifi, AF_INET, 
+					    &c->ip4.addr, c->ip4.prefix_len);
+			else
+				nl_addr_dup(c->ifi4, c->pasta_ifi, AF_INET);
+
 			nl_route(op_routes, c->ifi4, c->pasta_ifi, AF_INET,
 				 &c->ip4.gw);
 		}
 
 		if (c->ifi6) {
-			int prefix_len = 64;
-			nl_addr(op_addrs, c->ifi6, c->pasta_ifi, AF_INET6,
-				&c->ip6.addr, &prefix_len, NULL);
+			if (c->no_copy_addrs)
+				nl_addr_set(c->pasta_ifi, AF_INET6, &c->ip6.addr, 64);
+			else
+				nl_addr_dup(c->ifi4, c->pasta_ifi, AF_INET6);
+
 			nl_route(op_routes, c->ifi6, c->pasta_ifi, AF_INET6,
 				 &c->ip6.gw);
 		}
-- 
2.41.0

This improves consistency with IPv6 and makes it harder to misuse these as
some other sort of value.

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 netlink.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/netlink.c b/netlink.c
index 346eb3a..75d5988 100644
--- a/netlink.c
+++ b/netlink.c
@@ -265,9 +265,9 @@ void nl_route_set_def(unsigned int ifi, sa_family_t af, void *gw)
 			} r6;
 			struct {
 				struct rtattr rta_dst;
-				uint32_t d;
+				struct in_addr d;
 				struct rtattr rta_gw;
-				uint32_t a;
+				struct in_addr a;
 			} r4;
 		} set;
 	} req = {
@@ -309,7 +309,7 @@ void nl_route_set_def(unsigned int ifi, sa_family_t af, void *gw)
 		req.set.r4.rta_dst.rta_type = RTA_DST;
 		req.set.r4.rta_dst.rta_len = rta_len;
 
-		req.set.r4.a = *(uint32_t *)gw;
+		memcpy(&req.set.r4.a, gw, sizeof(req.set.r4.a));
 		req.set.r4.rta_gw.rta_type = RTA_GATEWAY;
 		req.set.r4.rta_gw.rta_len = rta_len;
 	}
@@ -471,9 +471,9 @@ void nl_addr_set(unsigned int ifi, sa_family_t af, void *addr, int
prefix_len)
 		union {
 			struct {
 				struct rtattr rta_l;
-				uint32_t l;
+				struct in_addr l;
 				struct rtattr rta_a;
-				uint32_t a;
+				struct in_addr a;
 			} a4;
 			struct {
 				struct rtattr rta_l;
@@ -517,7 +517,7 @@ void nl_addr_set(unsigned int ifi, sa_family_t af, void *addr, int
prefix_len)
 		req.nlh.nlmsg_len = offsetof(struct req_t, set.a4)
 			+ sizeof(req.set.a4);
 
-		req.set.a4.l = req.set.a4.a = *(uint32_t *)addr;
+		memcpy(&req.set.a4.l, addr, sizeof(req.set.a4.l));
 		req.set.a4.rta_l.rta_len = rta_len;
 		req.set.a4.rta_l.rta_type = IFA_LOCAL;
 		req.set.a4.rta_a.rta_len = rta_len;
-- 
2.41.0

nl_req() is designed to handle a single netlink request message: it only
receives a single reply datagram for the request, and only waits for a
single NLMSG_DONE or NLMSG_ERROR message at the beginning to clear out
things from previous requests.

However, in both nl_addr_dup() and nl_route_dup() we can send multiple
request messages as a single datagram, with a single nl_req() call.
This can easily mean that the replies nl_req() collects get out of
sync with requests.  We only get away with this because after we call
these functions we don't make any netlink calls where we need to parse
the replies.

This is fragile, so alter nl_*_dup() to make an nl_req() call for each
address it is adding in the target namespace.

For nl_route_dup() this fixes an additional minor problem: because
routes can have dependencies, some of the route add requests might
fail on the first attempt, so we need to repeat the requests a number
of times.  When we did that, we weren't updating the sequence number
on each new attempt.  This works, but not updating the sequence number
for each new request isn't ideal.  Now that we're making the requests
one at a time, it's easier to make sure we update the sequence number
each time.

Link: https://bugs.passt.top/show_bug.cgi?id=67

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 netlink.c | 50 +++++++++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/netlink.c b/netlink.c
index 72044cd..bd76098 100644
--- a/netlink.c
+++ b/netlink.c
@@ -351,18 +351,16 @@ void nl_route_dup(int s_src, unsigned int ifi_src,
 		.rta.rta_len	  = RTA_LENGTH(sizeof(unsigned int)),
 		.ifi		  = ifi_src,
 	};
-	char buf[NLBUFSIZ], resp[NLBUFSIZ];
 	unsigned dup_routes = 0;
 	ssize_t n, nlmsgs_size;
 	struct nlmsghdr *nh;
+	char buf[NLBUFSIZ];
 	unsigned i;
 
-	if ((n = nl_req(s_src, buf, &req, req.nlh.nlmsg_len)) < 0)
+	if ((nlmsgs_size = nl_req(s_src, buf, &req, req.nlh.nlmsg_len)) < 0)
 		return;
 
-	nlmsgs_size = n;
-
-	for (nh = (struct nlmsghdr *)buf;
+	for (nh = (struct nlmsghdr *)buf, n = nlmsgs_size;
 	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
 	     nh = NLMSG_NEXT(nh, n)) {
 		struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh);
@@ -372,7 +370,6 @@ void nl_route_dup(int s_src, unsigned int ifi_src,
 		if (nh->nlmsg_type != RTM_NEWROUTE)
 			continue;
 
-		nh->nlmsg_seq = nl_seq++;
 		nh->nlmsg_pid = 0;
 		nh->nlmsg_flags &= ~NLM_F_DUMP_FILTERED;
 		nh->nlmsg_flags |= NLM_F_REQUEST | NLM_F_ACK |
@@ -386,17 +383,27 @@ void nl_route_dup(int s_src, unsigned int ifi_src,
 		}
 	}
 
-	nh = (struct nlmsghdr *)buf;
 	/* Routes might have dependencies between each other, and the
 	 * kernel processes RTM_NEWROUTE messages sequentially. For n
-	 * valid routes, we might need to send up to n requests to get
-	 * all of them inserted. Routes that have been already
-	 * inserted won't cause the whole request to fail, so we can
-	 * simply repeat the whole request. This approach avoids the
-	 * need to calculate dependencies: let the kernel do that.
+	 * routes, we might need to send the requests up to n times to
+	 * get all of them inserted. Routes that have been already
+	 * inserted will return -EEXIST, but we can safely ignore that
+	 * and repeat the requests. This avoids the need to calculate
+	 * dependencies: let the kernel do that.
 	 */
-	for (i = 0; i < dup_routes; i++)
-		nl_req(s_dst, resp, nh, nlmsgs_size);
+	for (i = 0; i < dup_routes; i++) {
+		for (nh = (struct nlmsghdr *)buf, n = nlmsgs_size;
+		     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
+		     nh = NLMSG_NEXT(nh, n)) {
+			char resp[NLBUFSIZ];
+
+			if (nh->nlmsg_type != RTM_NEWROUTE)
+				continue;
+
+			nh->nlmsg_seq = nl_seq++;
+			nl_req(s_dst, resp, nh, nh->nlmsg_len);
+		}
+	}
 }
 
 /**
@@ -560,19 +567,18 @@ void nl_addr_dup(int s_src, unsigned int ifi_src,
 		.ifa.ifa_index     = ifi_src,
 		.ifa.ifa_prefixlen = 0,
 	};
-	char buf[NLBUFSIZ], resp[NLBUFSIZ];
-	ssize_t n, nlmsgs_size;
+	char buf[NLBUFSIZ];
 	struct nlmsghdr *nh;
+	ssize_t n;
 
 	if ((n = nl_req(s_src, buf, &req, sizeof(req))) < 0)
 		return;
 
-	nlmsgs_size = n;
-
 	for (nh = (struct nlmsghdr *)buf;
 	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
 	     nh = NLMSG_NEXT(nh, n)) {
 		struct ifaddrmsg *ifa;
+		char resp[NLBUFSIZ];
 		struct rtattr *rta;
 		size_t na;
 
@@ -587,10 +593,8 @@ void nl_addr_dup(int s_src, unsigned int ifi_src,
 		ifa = (struct ifaddrmsg *)NLMSG_DATA(nh);
 
 		if (ifa->ifa_scope == RT_SCOPE_LINK ||
-		    ifa->ifa_index != ifi_src) {
-			ifa->ifa_family = AF_UNSPEC;
+		    ifa->ifa_index != ifi_src)
 			continue;
-		}
 
 		ifa->ifa_index = ifi_dst;
 
@@ -599,9 +603,9 @@ void nl_addr_dup(int s_src, unsigned int ifi_src,
 			if (rta->rta_type == IFA_LABEL)
 				rta->rta_type = IFA_UNSPEC;
 		}
-	}
 
-	nl_req(s_dst, resp, buf, nlmsgs_size);
+		nl_req(s_dst, resp, nh, nh->nlmsg_len);
+	}
 }
 
 /**
-- 
2.41.0

Errors on send() or recv() calls on a netlink socket don't indicate errors
with the netlink operations we're attempting, but rather that something's
gone wrong with the mechanics of netlink itself.  We don't really expect
this to ever happen, and if it does, it's not clear what we could to to
recover.

So, treat errors from these calls as fatal, rather than returning the error
up the stack.  This makes handling failures in the callers of nl_req()
simpler.

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 netlink.c | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/netlink.c b/netlink.c
index 3620fd6..826c926 100644
--- a/netlink.c
+++ b/netlink.c
@@ -103,9 +103,9 @@ fail:
  * @req:	Request with netlink header
  * @len:	Request length
  *
- * Return: received length on success, negative error code on failure
+ * Return: received length on success, terminates on error
  */
-static int nl_req(int s, char *buf, const void *req, ssize_t len)
+static ssize_t nl_req(int s, char *buf, const void *req, ssize_t len)
 {
 	char flush[NLBUFSIZ];
 	int done = 0;
@@ -124,11 +124,17 @@ static int nl_req(int s, char *buf, const void *req, ssize_t len)
 		}
 	}
 
-	if ((send(s, req, len, 0) < len) ||
-	    (len = recv(s, buf, NLBUFSIZ, 0)) < 0)
-		return -errno;
+	n = send(s, req, len, 0);
+	if (n < 0)
+		die("netlink: Failed to send(): %s", strerror(errno));
+	else if (n < len)
+		die("netlink: Short send");
+
+	n = recv(s, buf, NLBUFSIZ, 0);
+	if (n < 0)
+		die("netlink: Failed to recv(): %s", strerror(errno));
 
-	return len;
+	return n;
 }
 
 /**
@@ -158,8 +164,7 @@ unsigned int nl_get_ext_if(int s, sa_family_t af)
 	ssize_t n;
 	size_t na;
 
-	if ((n = nl_req(s, buf, &req, sizeof(req))) < 0)
-		return 0;
+	n = nl_req(s, buf, &req, sizeof(req));
 
 	nh = (struct nlmsghdr *)buf;
 
@@ -218,8 +223,7 @@ void nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void
*gw)
 	char buf[NLBUFSIZ];
 	ssize_t n;
 
-	if ((n = nl_req(s, buf, &req, req.nlh.nlmsg_len)) < 0)
-		return;
+	n = nl_req(s, buf, &req, req.nlh.nlmsg_len);
 
 	for (nh = (struct nlmsghdr *)buf;
 	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
@@ -357,8 +361,7 @@ void nl_route_dup(int s_src, unsigned int ifi_src,
 	char buf[NLBUFSIZ];
 	unsigned i;
 
-	if ((nlmsgs_size = nl_req(s_src, buf, &req, req.nlh.nlmsg_len)) < 0)
-		return;
+	nlmsgs_size = nl_req(s_src, buf, &req, req.nlh.nlmsg_len);
 
 	for (nh = (struct nlmsghdr *)buf, n = nlmsgs_size;
 	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
@@ -434,8 +437,7 @@ void nl_addr_get(int s, unsigned int ifi, sa_family_t af,
 	char buf[NLBUFSIZ];
 	ssize_t n;
 
-	if ((n = nl_req(s, buf, &req, req.nlh.nlmsg_len)) < 0)
-		return;
+	n = nl_req(s, buf, &req, req.nlh.nlmsg_len);
 
 	for (nh = (struct nlmsghdr *)buf;
 	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
@@ -571,8 +573,7 @@ void nl_addr_dup(int s_src, unsigned int ifi_src,
 	struct nlmsghdr *nh;
 	ssize_t n;
 
-	if ((n = nl_req(s_src, buf, &req, sizeof(req))) < 0)
-		return;
+	n = nl_req(s_src, buf, &req, sizeof(req));
 
 	for (nh = (struct nlmsghdr *)buf;
 	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
@@ -632,9 +633,6 @@ void nl_link_get_mac(int s, unsigned int ifi, void *mac)
 	ssize_t n;
 
 	n = nl_req(s, buf, &req, sizeof(req));
-	if (n < 0)
-		return;
-
 	for (nh = (struct nlmsghdr *)buf;
 	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
 	     nh = NLMSG_NEXT(nh, n)) {
-- 
2.41.0

So far we never checked for errors reported on netlink operations via
NLMSG_ERROR messages.  This has led to several subtle and tricky to debug
situations which would have been obvious if we knew that certain netlink
operations had failed.

Introduce a nl_do() helper that performs netlink "do" operations (that is
making a single change without retreiving complex information) with much
more thorough error checking.  As well as returning an error code if we
get an NLMSG_ERROR message, we also check for unexpected behaviour in
several places.  That way if we've made a mistake in our assumptions about
how netlink works it should result in a clear error rather than some subtle
misbehaviour.

We update those calls to nl_req() that can use the new wrapper to do so.
We will extend those to better handle errors in future.  We don't touch
non-"do" operations for now, those are a bit trickier.

Link: https://bugs.passt.top/show_bug.cgi?id=60

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 netlink.c | 59 ++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 47 insertions(+), 12 deletions(-)

diff --git a/netlink.c b/netlink.c
index 3170344..cdd65c0 100644
--- a/netlink.c
+++ b/netlink.c
@@ -148,6 +148,47 @@ static ssize_t nl_req(int s, char *buf, void *req,
 	return n;
 }
 
+/**
+ * nl_do() - Send netlink "do" request, and wait for acknowledgement
+ * @s:		Netlink socket
+ * @req:	Request (will fill netlink header)
+ * @type:	Request type
+ * @flags:	Extra request flags (NLM_F_REQUEST and NLM_F_ACK assumed)
+ * @len:	Request length
+ *
+ * Return: 0 on success, negative error code on error
+ */
+static int nl_do(int s, void *req, uint16_t type, uint16_t flags, ssize_t len)
+{
+	struct nlmsghdr *nh;
+	char buf[NLBUFSIZ];
+	uint16_t seq;
+	ssize_t n;
+
+	n = nl_req(s, buf, req, type, flags, len);
+	seq = ((struct nlmsghdr *)req)->nlmsg_seq;
+
+	for (nh = (struct nlmsghdr *)buf;
+	     NLMSG_OK(nh, n); nh = NLMSG_NEXT(nh, n)) {
+		struct nlmsgerr *errmsg;
+
+		if (nh->nlmsg_seq != seq)
+			die("netlink: Unexpected response sequence number");
+
+		switch (nh->nlmsg_type) {
+		case NLMSG_DONE:
+			return 0;
+		case NLMSG_ERROR:
+			errmsg = (struct nlmsgerr *)NLMSG_DATA(nh);
+			return errmsg->error;
+		default:
+			warn("netlink: Unexpected response message");
+		}
+	}
+
+	die("netlink: Missing acknowledgement of request");
+}
+
 /**
  * nl_get_ext_if() - Get interface index supporting IP version being probed
  * @s:	Netlink socket
@@ -289,7 +330,6 @@ void nl_route_set_def(int s, unsigned int ifi, sa_family_t af, void
*gw)
 		.rta.rta_len	  = RTA_LENGTH(sizeof(unsigned int)),
 		.ifi		  = ifi,
 	};
-	char buf[NLBUFSIZ];
 	ssize_t len;
 
 	if (af == AF_INET6) {
@@ -316,7 +356,7 @@ void nl_route_set_def(int s, unsigned int ifi, sa_family_t af, void
*gw)
 		req.set.r4.rta_gw.rta_len = rta_len;
 	}
 
-	nl_req(s, buf, &req, RTM_NEWROUTE, NLM_F_CREATE | NLM_F_EXCL, len);
+	nl_do(s, &req, RTM_NEWROUTE, NLM_F_CREATE | NLM_F_EXCL, len);
 }
 
 /**
@@ -386,12 +426,11 @@ void nl_route_dup(int s_src, unsigned int ifi_src,
 		     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
 		     nh = NLMSG_NEXT(nh, n)) {
 			uint16_t flags = nh->nlmsg_flags;
-			char resp[NLBUFSIZ];
 
 			if (nh->nlmsg_type != RTM_NEWROUTE)
 				continue;
 
-			nl_req(s_dst, resp, nh, RTM_NEWROUTE,
+			nl_do(s_dst, nh, RTM_NEWROUTE,
 			       (flags & ~NLM_F_DUMP_FILTERED) | NLM_F_CREATE,
 			       nh->nlmsg_len);
 		}
@@ -490,7 +529,6 @@ void nl_addr_set(int s, unsigned int ifi, sa_family_t af,
 		.ifa.ifa_prefixlen = prefix_len,
 		.ifa.ifa_scope	   = RT_SCOPE_UNIVERSE,
 	};
-	char buf[NLBUFSIZ];
 	ssize_t len;
 
 	if (af == AF_INET6) {
@@ -519,7 +557,7 @@ void nl_addr_set(int s, unsigned int ifi, sa_family_t af,
 		req.set.a4.rta_a.rta_type = IFA_ADDRESS;
 	}
 
-	nl_req(s, buf, &req, RTM_NEWADDR, NLM_F_CREATE | NLM_F_EXCL, len);
+	nl_do(s, &req, RTM_NEWADDR, NLM_F_CREATE | NLM_F_EXCL, len);
 }
 
 /**
@@ -551,7 +589,6 @@ void nl_addr_dup(int s_src, unsigned int ifi_src,
 	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
 	     nh = NLMSG_NEXT(nh, n)) {
 		struct ifaddrmsg *ifa;
-		char resp[NLBUFSIZ];
 		struct rtattr *rta;
 		size_t na;
 
@@ -572,7 +609,7 @@ void nl_addr_dup(int s_src, unsigned int ifi_src,
 				rta->rta_type = IFA_UNSPEC;
 		}
 
-		nl_req(s_dst, resp, nh, RTM_NEWADDR,
+		nl_do(s_dst, nh, RTM_NEWADDR,
 		       (nh->nlmsg_flags & ~NLM_F_DUMP_FILTERED) | NLM_F_CREATE,
 		       nh->nlmsg_len);
 	}
@@ -640,11 +677,10 @@ void nl_link_set_mac(int s, unsigned int ifi, void *mac)
 		.rta.rta_type	  = IFLA_ADDRESS,
 		.rta.rta_len	  = RTA_LENGTH(ETH_ALEN),
 	};
-	char buf[NLBUFSIZ];
 
 	memcpy(req.mac, mac, ETH_ALEN);
 
-	nl_req(s, buf, &req, RTM_NEWLINK, 0, sizeof(req));
+	nl_do(s, &req, RTM_NEWLINK, 0, sizeof(req));
 }
 
 /**
@@ -670,11 +706,10 @@ void nl_link_up(int s, unsigned int ifi, int mtu)
 		.mtu		  = mtu,
 	};
 	ssize_t len = sizeof(req);
-	char buf[NLBUFSIZ];
 
 	if (!mtu)
 		/* Shorten request to drop MTU attribute */
 		len = offsetof(struct req_t, rta);
 
-	nl_req(s, buf, &req, RTM_NEWLINK, 0, len);
+	nl_do(s, &req, RTM_NEWLINK, 0, len);
 }
-- 
2.41.0

Currently nl_req() sends the request, and receives a single response
datagram which we then process.  However, a single request can result in
multiple response datagrams.  That happens nearly all the time for DUMP
requests, where the 'DONE' message usually comes in a second datagram after
the NEW{LINK|ADDR|ROUTE} messages.  It can also happen if there are just
too many objects to dump in a single datagram.

Allow our netlink code to process multiple response datagrams by splitting
nl_req() into three different helpers: nl_send() just sends a request,
without getting a response.  nl_status() checks a single message to see if
it indicates the end of the reponses for our request.  nl_next() moves onto
the next response message, whether it's in a datagram we already received
or we need to recv() a new one.  We also add a 'for'-style macro to use
these to step through every response message to a request across multiple
datagrams.

While we're at it, be more thourough with checking that our sequence
numbers are in sync.

Link: https://bugs.passt.top/show_bug.cgi?id=67

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 netlink.c | 181 ++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 113 insertions(+), 68 deletions(-)

diff --git a/netlink.c b/netlink.c
index d553ddd..9293e2b 100644
--- a/netlink.c
+++ b/netlink.c
@@ -104,18 +104,17 @@ fail:
 }
 
 /**
- * nl_req() - Prepare and send netlink request, read response
+ * nl_send() - Prepare and send netlink request
  * @s:		Netlink socket
- * @buf:	Buffer for response (at least NLBUFSIZ long)
  * @req:	Request (will fill netlink header)
  * @type:	Request type
  * @flags:	Extra request flags (NLM_F_REQUEST and NLM_F_ACK assumed)
  * @len:	Request length
  *
- * Return: received length on success, terminates on error
+ * Return: sequence number of request on success, terminates on error
  */
-static ssize_t nl_req(int s, char *buf, void *req,
-		      uint16_t type, uint16_t flags, ssize_t len)
+static uint16_t nl_send(int s, void *req, uint16_t type,
+		       uint16_t flags, ssize_t len)
 {
 	char flush[NLBUFSIZ];
 	struct nlmsghdr *nh;
@@ -148,13 +147,80 @@ static ssize_t nl_req(int s, char *buf, void *req,
 	else if (n < len)
 		die("netlink: Short send");
 
-	n = recv(s, buf, NLBUFSIZ, 0);
-	if (n < 0)
-		die("netlink: Failed to recv(): %s", strerror(errno));
+	return nh->nlmsg_seq;
+}
+
+/**
+ * nl_status() - Check status given by a netlink response
+ * @nh:		Netlink response header
+ * @n:		Remaining space in response buffer from @nh
+ * @seq:	Request sequence number we expect a response to
+ *
+ * Return: 0 if @nh indicated successful completion,
+ *         < 0, negative error code if @nh indicated failure
+ *         > 0 @n if there are more responses to request @seq
+ *     terminates if sequence numbers are out of sync
+ */
+static int nl_status(const struct nlmsghdr *nh, ssize_t n, uint16_t seq)
+{
+	ASSERT(NLMSG_OK(nh, n));
+
+	if (nh->nlmsg_seq != seq)
+		die("netlink: Unexpected sequence number (%hu != %hu)",
+		    nh->nlmsg_seq, seq);
+
+	if (nh->nlmsg_type == NLMSG_DONE) {
+		return 0;
+	}
+	if (nh->nlmsg_type == NLMSG_ERROR) {
+		struct nlmsgerr *errmsg = (struct nlmsgerr *)NLMSG_DATA(nh);
+		return errmsg->error;
+	}
 
 	return n;
 }
 
+/**
+ * nl_next() - Get next netlink response message, recv()ing if necessary
+ * @s:		Netlink socket
+ * @buf:	Buffer for responses (at least NLBUFSIZ long)
+ * @nh:		Previous message, or NULL if there are none
+ * @n:		Variable with remaining unread bytes in buffer (updated)
+ *
+ * Return: pointer to next unread netlink response message (may block)
+ */
+static struct nlmsghdr *nl_next(int s, char *buf, struct nlmsghdr *nh, ssize_t *n)
+{
+	if (nh) {
+		nh = NLMSG_NEXT(nh, *n);
+		if (NLMSG_OK(nh, *n))
+			return nh;
+	}
+
+	*n = recv(s, buf, NLBUFSIZ, 0);
+	if (*n < 0)
+		die("netlink: Failed to recv(): %s", strerror(errno));
+
+	nh = (struct nlmsghdr *)buf;
+	if (!NLMSG_OK(nh, *n))
+		die("netlink: Response datagram with no message");
+
+	return nh;
+}
+
+/**
+ * nl_foreach - 'for' type macro to step through netlink response messages
+ * @nh:		Steps through each response header (struct nlmsghdr *)
+ * @status:	When loop exits indicates if there was an error (ssize_t)
+ * @s:		Netlink socket
+ * @buf:	Buffer for responses (at least NLBUFSIZ long)
+ * @seq:	Sequence number of request we're getting responses for
+  */
+#define nl_foreach(nh, status, s, buf, seq)				\
+	for ((nh) = nl_next((s), (buf), NULL, &(status));		\
+	     ((status) = nl_status((nh), (status), (seq))) > 0;		\
+	     (nh) = nl_next((s), (buf), (nh), &(status)))
+
 /**
  * nl_do() - Send netlink "do" request, and wait for acknowledgement
  * @s:		Netlink socket
@@ -169,31 +235,14 @@ static int nl_do(int s, void *req, uint16_t type, uint16_t flags,
ssize_t len)
 {
 	struct nlmsghdr *nh;
 	char buf[NLBUFSIZ];
+	ssize_t status;
 	uint16_t seq;
-	ssize_t n;
-
-	n = nl_req(s, buf, req, type, flags, len);
-	seq = ((struct nlmsghdr *)req)->nlmsg_seq;
 
-	for (nh = (struct nlmsghdr *)buf;
-	     NLMSG_OK(nh, n); nh = NLMSG_NEXT(nh, n)) {
-		struct nlmsgerr *errmsg;
+	seq = nl_send(s, req, type, flags, len);
+	nl_foreach(nh, status, s, buf, seq)
+		warn("netlink: Unexpected response message");
 
-		if (nh->nlmsg_seq != seq)
-			die("netlink: Unexpected response sequence number");
-
-		switch (nh->nlmsg_type) {
-		case NLMSG_DONE:
-			return 0;
-		case NLMSG_ERROR:
-			errmsg = (struct nlmsgerr *)NLMSG_DATA(nh);
-			return errmsg->error;
-		default:
-			warn("netlink: Unexpected response message");
-		}
-	}
-
-	die("netlink: Missing acknowledgement of request");
+	return status;
 }
 
 /**
@@ -215,14 +264,12 @@ unsigned int nl_get_ext_if(int s, sa_family_t af)
 	struct nlmsghdr *nh;
 	struct rtattr *rta;
 	char buf[NLBUFSIZ];
-	ssize_t n;
+	ssize_t status;
+	uint16_t seq;
 	size_t na;
 
-	n = nl_req(s, buf, &req, RTM_GETROUTE, NLM_F_DUMP, sizeof(req));
-
-	nh = (struct nlmsghdr *)buf;
-
-	for ( ; NLMSG_OK(nh, n); nh = NLMSG_NEXT(nh, n)) {
+	seq = nl_send(s, &req, RTM_GETROUTE, NLM_F_DUMP, sizeof(req));
+	nl_foreach(nh, status, s, buf, seq) {
 		struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh);
 
 		if (rtm->rtm_dst_len || rtm->rtm_family != af)
@@ -270,13 +317,11 @@ void nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void
*gw)
 	};
 	struct nlmsghdr *nh;
 	char buf[NLBUFSIZ];
-	ssize_t n;
-
-	n = nl_req(s, buf, &req, RTM_GETROUTE, NLM_F_DUMP, sizeof(req));
+	ssize_t status;
+	uint16_t seq;
 
-	for (nh = (struct nlmsghdr *)buf;
-	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
-	     nh = NLMSG_NEXT(nh, n)) {
+	seq = nl_send(s, &req, RTM_GETROUTE, NLM_F_DUMP, sizeof(req));
+	nl_foreach(nh, status, s, buf, seq) {
 		struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh);
 		struct rtattr *rta;
 		size_t na;
@@ -392,18 +437,23 @@ void nl_route_dup(int s_src, unsigned int ifi_src,
 		.rta.rta_len	  = RTA_LENGTH(sizeof(unsigned int)),
 		.ifi		  = ifi_src,
 	};
+	ssize_t nlmsgs_size, status;
 	unsigned dup_routes = 0;
-	ssize_t n, nlmsgs_size;
 	struct nlmsghdr *nh;
 	char buf[NLBUFSIZ];
+	uint16_t seq;
 	unsigned i;
 
-	nlmsgs_size = nl_req(s_src, buf, &req,
-			     RTM_GETROUTE, NLM_F_DUMP, sizeof(req));
+	seq = nl_send(s_src, &req, RTM_GETROUTE, NLM_F_DUMP, sizeof(req));
 
-	for (nh = (struct nlmsghdr *)buf, n = nlmsgs_size;
-	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
-	     nh = NLMSG_NEXT(nh, n)) {
+	/* nl_foreach() will step through multiple response datagrams,
+	 * which we don't want here because we need to have all the
+	 * routes in the buffer at once.
+	 */
+	nh = nl_next(s_src, buf, NULL, &nlmsgs_size);
+	for (status = nlmsgs_size;
+	     NLMSG_OK(nh, status) && (status = nl_status(nh, status, seq)) > 0;
+	     nh = NLMSG_NEXT(nh, status)) {
 		struct rtmsg *rtm = (struct rtmsg *)NLMSG_DATA(nh);
 		struct rtattr *rta;
 		size_t na;
@@ -429,9 +479,9 @@ void nl_route_dup(int s_src, unsigned int ifi_src,
 	 * dependencies: let the kernel do that.
 	 */
 	for (i = 0; i < dup_routes; i++) {
-		for (nh = (struct nlmsghdr *)buf, n = nlmsgs_size;
-		     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
-		     nh = NLMSG_NEXT(nh, n)) {
+		for (nh = (struct nlmsghdr *)buf, status = nlmsgs_size;
+		     NLMSG_OK(nh, status);
+		     nh = NLMSG_NEXT(nh, status)) {
 			uint16_t flags = nh->nlmsg_flags;
 
 			if (nh->nlmsg_type != RTM_NEWROUTE)
@@ -465,13 +515,11 @@ void nl_addr_get(int s, unsigned int ifi, sa_family_t af,
 	};
 	struct nlmsghdr *nh;
 	char buf[NLBUFSIZ];
-	ssize_t n;
-
-	n = nl_req(s, buf, &req, RTM_GETADDR, NLM_F_DUMP, sizeof(req));
+	ssize_t status;
+	uint16_t seq;
 
-	for (nh = (struct nlmsghdr *)buf;
-	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
-	     nh = NLMSG_NEXT(nh, n)) {
+	seq = nl_send(s, &req, RTM_GETADDR, NLM_F_DUMP, sizeof(req));
+	nl_foreach(nh, status, s, buf, seq) {
 		struct ifaddrmsg *ifa = (struct ifaddrmsg *)NLMSG_DATA(nh);
 		struct rtattr *rta;
 		size_t na;
@@ -588,13 +636,11 @@ void nl_addr_dup(int s_src, unsigned int ifi_src,
 	};
 	char buf[NLBUFSIZ];
 	struct nlmsghdr *nh;
-	ssize_t n;
-
-	n = nl_req(s_src, buf, &req, RTM_GETADDR, NLM_F_DUMP, sizeof(req));
+	ssize_t status;
+	uint16_t seq;
 
-	for (nh = (struct nlmsghdr *)buf;
-	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
-	     nh = NLMSG_NEXT(nh, n)) {
+	seq = nl_send(s_src, &req, RTM_GETADDR, NLM_F_DUMP, sizeof(req));
+	nl_foreach(nh, status, s_src, buf, seq) {
 		struct ifaddrmsg *ifa;
 		struct rtattr *rta;
 		size_t na;
@@ -639,12 +685,11 @@ void nl_link_get_mac(int s, unsigned int ifi, void *mac)
 	};
 	struct nlmsghdr *nh;
 	char buf[NLBUFSIZ];
-	ssize_t n;
+	ssize_t status;
+	uint16_t seq;
 
-	n = nl_req(s, buf, &req, RTM_GETLINK, 0, sizeof(req));
-	for (nh = (struct nlmsghdr *)buf;
-	     NLMSG_OK(nh, n) && nh->nlmsg_type != NLMSG_DONE;
-	     nh = NLMSG_NEXT(nh, n)) {
+	seq = nl_send(s, &req, RTM_GETLINK, 0, sizeof(req));
+	nl_foreach(nh, status, s, buf, seq) {
 		struct ifinfomsg *ifm = (struct ifinfomsg *)NLMSG_DATA(nh);
 		struct rtattr *rta;
 		size_t na;
-- 
2.41.0

Currently if anything goes wrong while we're configuring the namespace
network with --config-net, we'll just ignore it and carry on.  This might
lead to a silently unconfigured or misconfigured namespace environment.

For simple "set" operations based on nl_do() we can now detect failures
reported via netlink.  Propagate those errors up to pasta_ns_conf() and
report them usefully.

Link: https://bugs.passt.top/show_bug.cgi?id=60

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 netlink.c | 26 +++++++++++++++++---------
 netlink.h | 10 +++++-----
 pasta.c   | 42 ++++++++++++++++++++++++++++++++----------
 3 files changed, 54 insertions(+), 24 deletions(-)

diff --git a/netlink.c b/netlink.c
index 1d40856..4932f07 100644
--- a/netlink.c
+++ b/netlink.c
@@ -354,8 +354,10 @@ void nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void
*gw)
  * @ifi:	Interface index in target namespace
  * @af:		Address family
  * @gw:		Default gateway to set
+ *
+ * Return: 0 on success, negative error code on failure
  */
-void nl_route_set_def(int s, unsigned int ifi, sa_family_t af, void *gw)
+int nl_route_set_def(int s, unsigned int ifi, sa_family_t af, void *gw)
 {
 	struct req_t {
 		struct nlmsghdr nlh;
@@ -413,7 +415,7 @@ void nl_route_set_def(int s, unsigned int ifi, sa_family_t af, void
*gw)
 		req.set.r4.rta_gw.rta_len = rta_len;
 	}
 
-	nl_do(s, &req, RTM_NEWROUTE, NLM_F_CREATE | NLM_F_EXCL, len);
+	return nl_do(s, &req, RTM_NEWROUTE, NLM_F_CREATE | NLM_F_EXCL, len);
 }
 
 /**
@@ -559,9 +561,11 @@ void nl_addr_get(int s, unsigned int ifi, sa_family_t af,
  * @af:		Address family
  * @addr:	Global address to set
  * @prefix_len:	Mask or prefix length to set
+ *
+ * Return: 0 on success, negative error code on failure
  */
-void nl_addr_set(int s, unsigned int ifi, sa_family_t af,
-		 void *addr, int prefix_len)
+int nl_addr_set(int s, unsigned int ifi, sa_family_t af,
+		void *addr, int prefix_len)
 {
 	struct req_t {
 		struct nlmsghdr nlh;
@@ -614,7 +618,7 @@ void nl_addr_set(int s, unsigned int ifi, sa_family_t af,
 		req.set.a4.rta_a.rta_type = IFA_ADDRESS;
 	}
 
-	nl_do(s, &req, RTM_NEWADDR, NLM_F_CREATE | NLM_F_EXCL, len);
+	return nl_do(s, &req, RTM_NEWADDR, NLM_F_CREATE | NLM_F_EXCL, len);
 }
 
 /**
@@ -714,8 +718,10 @@ void nl_link_get_mac(int s, unsigned int ifi, void *mac)
  * @ns:		Use netlink socket in namespace
  * @ifi:	Interface index
  * @mac:	MAC address to set
+ *
+ * Return: 0 on success, negative error code on failure
  */
-void nl_link_set_mac(int s, unsigned int ifi, void *mac)
+int nl_link_set_mac(int s, unsigned int ifi, void *mac)
 {
 	struct req_t {
 		struct nlmsghdr nlh;
@@ -731,7 +737,7 @@ void nl_link_set_mac(int s, unsigned int ifi, void *mac)
 
 	memcpy(req.mac, mac, ETH_ALEN);
 
-	nl_do(s, &req, RTM_NEWLINK, 0, sizeof(req));
+	return nl_do(s, &req, RTM_NEWLINK, 0, sizeof(req));
 }
 
 /**
@@ -739,8 +745,10 @@ void nl_link_set_mac(int s, unsigned int ifi, void *mac)
  * @s:		Netlink socket
  * @ifi:	Interface index
  * @mtu:	If non-zero, set interface MTU
+ *
+ * Return: 0 on success, negative error code on failure
  */
-void nl_link_up(int s, unsigned int ifi, int mtu)
+int nl_link_up(int s, unsigned int ifi, int mtu)
 {
 	struct req_t {
 		struct nlmsghdr nlh;
@@ -762,5 +770,5 @@ void nl_link_up(int s, unsigned int ifi, int mtu)
 		/* Shorten request to drop MTU attribute */
 		len = offsetof(struct req_t, rta);
 
-	nl_do(s, &req, RTM_NEWLINK, 0, len);
+	return nl_do(s, &req, RTM_NEWLINK, 0, len);
 }
diff --git a/netlink.h b/netlink.h
index 5ca17c6..977244b 100644
--- a/netlink.h
+++ b/netlink.h
@@ -12,17 +12,17 @@ extern int nl_sock_ns;
 void nl_sock_init(const struct ctx *c, bool ns);
 unsigned int nl_get_ext_if(int s, sa_family_t af);
 void nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void *gw);
-void nl_route_set_def(int s, unsigned int ifi, sa_family_t af, void *gw);
+int nl_route_set_def(int s, unsigned int ifi, sa_family_t af, void *gw);
 void nl_route_dup(int s_src, unsigned int ifi_src,
 		  int s_dst, unsigned int ifi_dst, sa_family_t af);
 void nl_addr_get(int s, unsigned int ifi, sa_family_t af,
 		 void *addr, int *prefix_len, void *addr_l);
-void nl_addr_set(int s, unsigned int ifi, sa_family_t af,
-		 void *addr, int prefix_len);
+int nl_addr_set(int s, unsigned int ifi, sa_family_t af,
+		void *addr, int prefix_len);
 void nl_addr_dup(int s_src, unsigned int ifi_src,
 		 int s_dst, unsigned int ifi_dst, sa_family_t af);
 void nl_link_get_mac(int s, unsigned int ifi, void *mac);
-void nl_link_set_mac(int s, unsigned int ifi, void *mac);
-void nl_link_up(int s, unsigned int ifi, int mtu);
+int nl_link_set_mac(int s, unsigned int ifi, void *mac);
+int nl_link_up(int s, unsigned int ifi, int mtu);
 
 #endif /* NETLINK_H */
diff --git a/pasta.c b/pasta.c
index 3380475..ed6fda3 100644
--- a/pasta.c
+++ b/pasta.c
@@ -272,49 +272,71 @@ void pasta_start_ns(struct ctx *c, uid_t uid, gid_t gid,
  */
 void pasta_ns_conf(struct ctx *c)
 {
-	nl_link_up(nl_sock_ns, 1 /* lo */, 0);
+	int rc = 0;
+
+	rc = nl_link_up(nl_sock_ns, 1 /* lo */, 0);
+	if (rc < 0)
+		die("Couldn't bring up loopback interface in namespace: %s",
+		    strerror(-rc));
 
 	/* Get or set guest MAC */
 	if (MAC_IS_ZERO(c->mac_guest))
 		nl_link_get_mac(nl_sock_ns, c->pasta_ifi, c->mac_guest);
 	else
-		nl_link_set_mac(nl_sock_ns, c->pasta_ifi, c->mac_guest);
+		rc = nl_link_set_mac(nl_sock_ns, c->pasta_ifi, c->mac_guest);
+	if (rc < 0)
+		die("Couldn't set MAC address in namespace: %s",
+		    strerror(-rc));
 
 	if (c->pasta_conf_ns) {
 		nl_link_up(nl_sock_ns, c->pasta_ifi, c->mtu);
 
 		if (c->ifi4) {
 			if (c->no_copy_addrs)
-				nl_addr_set(nl_sock_ns, c->pasta_ifi, AF_INET,
-					    &c->ip4.addr, c->ip4.prefix_len);
+				rc = nl_addr_set(nl_sock_ns, c->pasta_ifi,
+						 AF_INET,
+						 &c->ip4.addr,
+						 c->ip4.prefix_len);
 			else
 				nl_addr_dup(nl_sock, c->ifi4,
 					    nl_sock_ns, c->pasta_ifi, AF_INET);
+			if (rc < 0)
+				die("Couldn't set IPv4 address(es) in namespace: %s",
+				    strerror(-rc));
 
 			if (c->no_copy_routes)
-				nl_route_set_def(nl_sock_ns, c->pasta_ifi,
-						 AF_INET, &c->ip4.gw);
+				rc = nl_route_set_def(nl_sock_ns, c->pasta_ifi,
+						      AF_INET, &c->ip4.gw);
 			else
 				nl_route_dup(nl_sock, c->ifi4, nl_sock_ns,
 					     c->pasta_ifi, AF_INET);
+			if (rc < 0)
+				die("Couldn't set IPv4 route(s) in guest: %s",
+				    strerror(-rc));
 		}
 
 		if (c->ifi6) {
 			if (c->no_copy_addrs)
-				nl_addr_set(nl_sock_ns, c->pasta_ifi,
-					    AF_INET6, &c->ip6.addr, 64);
+				rc = nl_addr_set(nl_sock_ns, c->pasta_ifi,
+						 AF_INET6, &c->ip6.addr, 64);
 			else
 				nl_addr_dup(nl_sock, c->ifi4,
 					    nl_sock_ns, c->pasta_ifi,
 					    AF_INET6);
+			if (rc < 0)
+				die("Couldn't set IPv6 address(es) in namespace: %s",
+				    strerror(-rc));
 
 			if (c->no_copy_routes)
-				nl_route_set_def(nl_sock_ns, c->pasta_ifi,
-						 AF_INET6, &c->ip6.gw);
+				rc = nl_route_set_def(nl_sock_ns, c->pasta_ifi,
+						      AF_INET6, &c->ip6.gw);
 			else
 				nl_route_dup(nl_sock, c->ifi6,
 					     nl_sock_ns, c->pasta_ifi,
 					     AF_INET6);
+			if (rc < 0)
+				die("Couldn't set IPv6 route(s) in guest: %s",
+				    strerror(-rc));
 		}
 	}
 
-- 
2.41.0

Currently if we receive any netlink errors while discovering network
configuration from the host, we'll just ignore it and carry on.  This
might lead to cryptic error messages later on, or even silent
misconfiguration.

We now have the mechanisms to detect errors from get/dump netlink
operations.  Propgate these errors up to the callers and report them usefully.

Link: https://bugs.passt.top/show_bug.cgi?id=60

Signed-off-by: David Gibson <david(a)gibson.dropbear.id.au>
---
 conf.c    | 65 +++++++++++++++++++++++++++++++++++++++++++------------
 netlink.c | 19 ++++++++++++----
 netlink.h |  8 +++----
 3 files changed, 70 insertions(+), 22 deletions(-)

diff --git a/conf.c b/conf.c
index 2e6e03f..14ec9f3 100644
--- a/conf.c
+++ b/conf.c
@@ -647,12 +647,24 @@ static unsigned int conf_ip4(unsigned int ifi,
 		return 0;
 	}
 
-	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->gw))
-		nl_route_get_def(nl_sock, ifi, AF_INET, &ip4->gw);
+	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->gw)) {
+		int rc = nl_route_get_def(nl_sock, ifi, AF_INET, &ip4->gw);
+		if (rc < 0) {
+			err("Couldn't discover IPv4 gateway address: %s",
+			    strerror(-rc));
+			return 0;
+		}
+	}
 
-	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr))
-		nl_addr_get(nl_sock, ifi, AF_INET,
-			    &ip4->addr, &ip4->prefix_len, NULL);
+	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr)) {
+		int rc = nl_addr_get(nl_sock, ifi, AF_INET,
+				     &ip4->addr, &ip4->prefix_len, NULL);
+		if (rc < 0) {
+			err("Couldn't discover IPv4 address: %s",
+			    strerror(-rc));
+			return 0;
+		}
+	}
 
 	if (!ip4->prefix_len) {
 		in_addr_t addr = ntohl(ip4->addr.s_addr);
@@ -668,8 +680,15 @@ static unsigned int conf_ip4(unsigned int ifi,
 
 	memcpy(&ip4->addr_seen, &ip4->addr, sizeof(ip4->addr_seen));
 
-	if (MAC_IS_ZERO(mac))
-		nl_link_get_mac(nl_sock, ifi, mac);
+	if (MAC_IS_ZERO(mac)) {
+		int rc = nl_link_get_mac(nl_sock, ifi, mac);
+		if (rc < 0) {
+			char ifname[IFNAMSIZ];
+			err("Couldn't discover MAC for %s: %s",
+			    if_indextoname(ifi, ifname), strerror(-rc));
+			return 0;
+		}
+	}
 
 	if (IN4_IS_ADDR_UNSPECIFIED(&ip4->addr) ||
 	    MAC_IS_ZERO(mac))
@@ -690,6 +709,7 @@ static unsigned int conf_ip6(unsigned int ifi,
 			     struct ip6_ctx *ip6, unsigned char *mac)
 {
 	int prefix_len = 0;
+	int rc;
 
 	if (!ifi)
 		ifi = nl_get_ext_if(nl_sock, AF_INET6);
@@ -699,18 +719,35 @@ static unsigned int conf_ip6(unsigned int ifi,
 		return 0;
 	}
 
-	if (IN6_IS_ADDR_UNSPECIFIED(&ip6->gw))
-		nl_route_get_def(nl_sock, ifi, AF_INET6, &ip6->gw);
+	if (IN6_IS_ADDR_UNSPECIFIED(&ip6->gw)) {
+		rc = nl_route_get_def(nl_sock, ifi, AF_INET6, &ip6->gw);
+		if (rc < 0) {
+			err("Couldn't discover IPv6 gateway address: %s",
+			    strerror(-rc));
+			return 0;
+		}
+	}
 
-	nl_addr_get(nl_sock, ifi, AF_INET6,
-		    IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) ? &ip6->addr : NULL,
-		    &prefix_len, &ip6->addr_ll);
+	rc = nl_addr_get(nl_sock, ifi, AF_INET6,
+			 IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) ? &ip6->addr : NULL,
+			 &prefix_len, &ip6->addr_ll);
+	if (rc < 0) {
+		err("Couldn't discover IPv6 address: %s", strerror(-rc));
+		return 0;
+	}
 
 	memcpy(&ip6->addr_seen, &ip6->addr, sizeof(ip6->addr));
 	memcpy(&ip6->addr_ll_seen, &ip6->addr_ll, sizeof(ip6->addr_ll));
 
-	if (MAC_IS_ZERO(mac))
-		nl_link_get_mac(0, ifi, mac);
+	if (MAC_IS_ZERO(mac)) {
+		rc = nl_link_get_mac(nl_sock, ifi, mac);
+		if (rc < 0) {
+			char ifname[IFNAMSIZ];
+			err("Couldn't discover MAC for %s: %s",
+			    if_indextoname(ifi, ifname), strerror(-rc));
+			return 0;
+		}
+	}
 
 	if (IN6_IS_ADDR_UNSPECIFIED(&ip6->addr) ||
 	    IN6_IS_ADDR_UNSPECIFIED(&ip6->addr_ll) ||
diff --git a/netlink.c b/netlink.c
index c57ee70..9e72b16 100644
--- a/netlink.c
+++ b/netlink.c
@@ -277,6 +277,8 @@ unsigned int nl_get_ext_if(int s, sa_family_t af)
 			ifi = *(unsigned int *)RTA_DATA(rta);
 		}
 	}
+	if (status < 0)
+		warn("netlink: RTM_GETROUTE failed: %s", strerror(-status));
 
 	return ifi;
 }
@@ -287,8 +289,10 @@ unsigned int nl_get_ext_if(int s, sa_family_t af)
  * @ifi:	Interface index
  * @af:		Address family
  * @gw:		Default gateway to fill on NL_GET
+ *
+ * Return: 0 on success, negative error code on failure
  */
-void nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void *gw)
+int nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void *gw)
 {
 	struct req_t {
 		struct nlmsghdr nlh;
@@ -329,6 +333,7 @@ void nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void
*gw)
 			found = true;
 		}
 	}
+	return status;
 }
 
 /**
@@ -508,9 +513,11 @@ void nl_route_dup(int s_src, unsigned int ifi_src,
  * @addr:	Global address to fill
  * @prefix_len:	Mask or prefix length, to fill (for IPv4)
  * @addr_l:	Link-scoped address to fill (for IPv6)
+ *
+ * Return: 9 on success, negative error code on failure
  */
-void nl_addr_get(int s, unsigned int ifi, sa_family_t af,
-		 void *addr, int *prefix_len, void *addr_l)
+int nl_addr_get(int s, unsigned int ifi, sa_family_t af,
+		void *addr, int *prefix_len, void *addr_l)
 {
 	struct req_t {
 		struct nlmsghdr nlh;
@@ -551,6 +558,7 @@ void nl_addr_get(int s, unsigned int ifi, sa_family_t af,
 				memcpy(addr_l, RTA_DATA(rta), RTA_PAYLOAD(rta));
 		}
 	}
+	return status;
 }
 
 /**
@@ -678,8 +686,10 @@ void nl_addr_dup(int s_src, unsigned int ifi_src,
  * @s:		Netlink socket
  * @ifi:	Interface index
  * @mac:	Fill with current MAC address
+ *
+ * Return: 0 on success, negative error code on failure
  */
-void nl_link_get_mac(int s, unsigned int ifi, void *mac)
+int nl_link_get_mac(int s, unsigned int ifi, void *mac)
 {
 	struct req_t {
 		struct nlmsghdr nlh;
@@ -708,6 +718,7 @@ void nl_link_get_mac(int s, unsigned int ifi, void *mac)
 			memcpy(mac, RTA_DATA(rta), ETH_ALEN);
 		}
 	}
+	return status;
 }
 
 /**
diff --git a/netlink.h b/netlink.h
index 977244b..b831405 100644
--- a/netlink.h
+++ b/netlink.h
@@ -11,17 +11,17 @@ extern int nl_sock_ns;
 
 void nl_sock_init(const struct ctx *c, bool ns);
 unsigned int nl_get_ext_if(int s, sa_family_t af);
-void nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void *gw);
+int nl_route_get_def(int s, unsigned int ifi, sa_family_t af, void *gw);
 int nl_route_set_def(int s, unsigned int ifi, sa_family_t af, void *gw);
 void nl_route_dup(int s_src, unsigned int ifi_src,
 		  int s_dst, unsigned int ifi_dst, sa_family_t af);
-void nl_addr_get(int s, unsigned int ifi, sa_family_t af,
-		 void *addr, int *prefix_len, void *addr_l);
+int nl_addr_get(int s, unsigned int ifi, sa_family_t af,
+		void *addr, int *prefix_len, void *addr_l);
 int nl_addr_set(int s, unsigned int ifi, sa_family_t af,
 		void *addr, int prefix_len);
 void nl_addr_dup(int s_src, unsigned int ifi_src,
 		 int s_dst, unsigned int ifi_dst, sa_family_t af);
-void nl_link_get_mac(int s, unsigned int ifi, void *mac);
+int nl_link_get_mac(int s, unsigned int ifi, void *mac);
 int nl_link_set_mac(int s, unsigned int ifi, void *mac);
 int nl_link_up(int s, unsigned int ifi, int mtu);
 
-- 
2.41.0