linux/drivers/net/netkit.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2023 Isovalent */

#include <linux/netdevice.h>
#include <linux/ethtool.h>
#include <linux/etherdevice.h>
#include <linux/filter.h>
#include <linux/netfilter_netdev.h>
#include <linux/bpf_mprog.h>
#include <linux/indirect_call_wrapper.h>

#include <net/netdev_lock.h>
#include <net/netdev_queues.h>
#include <net/netdev_rx_queue.h>
#include <net/xdp_sock_drv.h>
#include <net/netkit.h>
#include <net/dst.h>
#include <net/tcx.h>

#define NETKIT_DRV_NAME	"netkit"

#define NETKIT_NUM_RX_QUEUES_MAX  1024
#define NETKIT_NUM_TX_QUEUES_MAX  1

#define NETKIT_NUM_RX_QUEUES_REAL 1
#define NETKIT_NUM_TX_QUEUES_REAL 1

struct netkit {
	__cacheline_group_begin(netkit_fastpath);
	struct net_device __rcu *peer;
	struct bpf_mprog_entry __rcu *active;
	enum netkit_action policy;
	enum netkit_scrub scrub;
	struct bpf_mprog_bundle	bundle;
	__cacheline_group_end(netkit_fastpath);

	__cacheline_group_begin(netkit_slowpath);
	enum netkit_mode mode;
	enum netkit_pairing pair;
	bool primary;
	u32 headroom;
	__cacheline_group_end(netkit_slowpath);
};

struct netkit_link {
	struct bpf_link link;
	struct net_device *dev;
};

static struct rtnl_link_ops netkit_link_ops;

static __always_inline int
netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
	   enum netkit_action ret)
{
	const struct bpf_mprog_fp *fp;
	const struct bpf_prog *prog;

	bpf_mprog_foreach_prog(entry, fp, prog) {
		bpf_compute_data_pointers(skb);
		ret = bpf_prog_run(prog, skb);
		if (ret != NETKIT_NEXT)
			break;
	}
	return ret;
}

static void netkit_xnet(struct sk_buff *skb)
{
	skb->priority = 0;
	skb->mark = 0;
}

static void netkit_prep_forward(struct sk_buff *skb,
				bool xnet, bool xnet_scrub)
{
	skb_scrub_packet(skb, false);
	nf_skip_egress(skb, true);
	skb_reset_mac_header(skb);
	if (!xnet)
		return;
	skb_clear_tstamp(skb);
	if (xnet_scrub)
		netkit_xnet(skb);
}

static struct netkit *netkit_priv(const struct net_device *dev)
{
	return netdev_priv(dev);
}

static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
	struct netkit *nk = netkit_priv(dev);
	enum netkit_action ret = READ_ONCE(nk->policy);
	netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
	const struct bpf_mprog_entry *entry;
	struct net_device *peer;
	int len = skb->len;

	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
	rcu_read_lock();
	peer = rcu_dereference(nk->peer);
	if (unlikely(!peer || !(peer->flags & IFF_UP) ||
		     !pskb_may_pull(skb, ETH_HLEN) ||
		     skb_orphan_frags(skb, GFP_ATOMIC)))
		goto drop;
	netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)),
			    nk->scrub);
	eth_skb_pkt_type(skb, peer);
	skb->dev = peer;
	entry = rcu_dereference(nk->active);
	if (entry)
		ret = netkit_run(entry, skb, ret);
	switch (ret) {
	case NETKIT_NEXT:
	case NETKIT_PASS:
		eth_skb_pull_mac(skb);
		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
		if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
			dev_sw_netstats_tx_add(dev, 1, len);
			dev_sw_netstats_rx_add(peer, len);
		} else {
			goto drop_stats;
		}
		break;
	case NETKIT_REDIRECT:
		dev_sw_netstats_tx_add(dev, 1, len);
		skb_do_redirect(skb);
		break;
	case NETKIT_DROP:
	default:
drop:
		kfree_skb(skb);
drop_stats:
		dev_core_stats_tx_dropped_inc(dev);
		ret_dev = NET_XMIT_DROP;
		break;
	}
	rcu_read_unlock();
	bpf_net_ctx_clear(bpf_net_ctx);
	return ret_dev;
}

static int netkit_open(struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer = rtnl_dereference(nk->peer);

	if (nk->pair == NETKIT_DEVICE_SINGLE) {
		netif_carrier_on(dev);
		return 0;
	}
	if (!peer)
		return -ENOTCONN;
	if (peer->flags & IFF_UP) {
		netif_carrier_on(dev);
		netif_carrier_on(peer);
	}
	return 0;
}

static int netkit_close(struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer = rtnl_dereference(nk->peer);

	netif_carrier_off(dev);
	if (peer)
		netif_carrier_off(peer);
	return 0;
}

static int netkit_get_iflink(const struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer;
	int iflink = 0;

	rcu_read_lock();
	peer = rcu_dereference(nk->peer);
	if (peer)
		iflink = READ_ONCE(peer->ifindex);
	rcu_read_unlock();
	return iflink;
}

static void netkit_set_multicast(struct net_device *dev,
				 struct netdev_hw_addr_list *uc,
				 struct netdev_hw_addr_list *mc)
{
	/* Nothing to do, we receive whatever gets pushed to us! */
}

static int netkit_set_macaddr(struct net_device *dev, void *sa)
{
	struct netkit *nk = netkit_priv(dev);

	if (nk->mode != NETKIT_L2)
		return -EOPNOTSUPP;

	return eth_mac_addr(dev, sa);
}

static void netkit_set_headroom(struct net_device *dev, int headroom)
{
	struct netkit *nk = netkit_priv(dev), *nk2;
	struct net_device *peer;

	if (headroom < 0)
		headroom = NET_SKB_PAD;

	rcu_read_lock();
	peer = rcu_dereference(nk->peer);
	if (!peer) {
		nk->headroom = headroom;
		dev->needed_headroom = headroom;
	} else {
		nk2 = netkit_priv(peer);
		nk->headroom = headroom;
		headroom = max(nk->headroom, nk2->headroom);

		peer->needed_headroom = headroom;
		dev->needed_headroom = headroom;
	}
	rcu_read_unlock();
}

INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
{
	return rcu_dereference(netkit_priv(dev)->peer);
}

static void netkit_get_stats(struct net_device *dev,
			     struct rtnl_link_stats64 *stats)
{
	dev_fetch_sw_netstats(stats, dev->tstats);
	stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
}

static bool netkit_xsk_supported_at_phys(const struct net_device *dev)
{
	if (!dev->netdev_ops->ndo_bpf ||
	    !dev->netdev_ops->ndo_xdp_xmit ||
	    !dev->netdev_ops->ndo_xsk_wakeup)
		return false;
	return true;
}

static int netkit_xsk(struct net_device *dev, struct netdev_bpf *xdp)
{
	struct netkit *nk = netkit_priv(dev);
	struct netdev_bpf xdp_lower;
	struct netdev_rx_queue *rxq;
	struct net_device *phys;
	bool create = false;
	int ret = -EBUSY;

	switch (xdp->command) {
	case XDP_SETUP_XSK_POOL:
		if (nk->pair == NETKIT_DEVICE_PAIR)
			return -EOPNOTSUPP;
		if (xdp->xsk.queue_id >= dev->real_num_rx_queues)
			return -EINVAL;

		rxq = __netif_get_rx_queue(dev, xdp->xsk.queue_id);
		if (!rxq->lease)
			return -EOPNOTSUPP;

		phys = rxq->lease->dev;
		if (!netkit_xsk_supported_at_phys(phys))
			return -EOPNOTSUPP;

		create = xdp->xsk.pool;
		memcpy(&xdp_lower, xdp, sizeof(xdp_lower));
		xdp_lower.xsk.queue_id = get_netdev_rx_queue_index(rxq->lease);
		break;
	case XDP_SETUP_PROG:
		return -EOPNOTSUPP;
	default:
		return -EINVAL;
	}

	netdev_lock(phys);
	if (create &&
	    (phys->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) {
		ret = -EOPNOTSUPP;
		goto out;
	}
	if (!create || !dev_get_min_mp_channel_count(phys))
		ret = phys->netdev_ops->ndo_bpf(phys, &xdp_lower);
out:
	netdev_unlock(phys);
	return ret;
}

static int netkit_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
{
	struct netdev_rx_queue *rxq, *rxq_lease;
	struct net_device *phys;

	if (queue_id >= dev->real_num_rx_queues)
		return -EINVAL;

	rxq = __netif_get_rx_queue(dev, queue_id);
	rxq_lease = READ_ONCE(rxq->lease);
	if (unlikely(!rxq_lease))
		return -EOPNOTSUPP;

	/* netkit_xsk already validated full xsk support, hence it's
	 * fine to call into ndo_xsk_wakeup right away given this
	 * was a prerequisite to get here in the first place. The
	 * phys xsk support cannot change without tearing down the
	 * device (which clears the lease first).
	 */
	phys = rxq_lease->dev;
	return phys->netdev_ops->ndo_xsk_wakeup(phys,
			get_netdev_rx_queue_index(rxq_lease), flags);
}

static int netkit_init(struct net_device *dev)
{
	netdev_lockdep_set_classes(dev);
	return 0;
}

static void netkit_uninit(struct net_device *dev);

static const struct net_device_ops netkit_netdev_ops = {
	.ndo_init		= netkit_init,
	.ndo_open		= netkit_open,
	.ndo_stop		= netkit_close,
	.ndo_start_xmit		= netkit_xmit,
	.ndo_set_rx_mode_async	= netkit_set_multicast,
	.ndo_set_rx_headroom	= netkit_set_headroom,
	.ndo_set_mac_address	= netkit_set_macaddr,
	.ndo_get_iflink		= netkit_get_iflink,
	.ndo_get_peer_dev	= netkit_peer_dev,
	.ndo_get_stats64	= netkit_get_stats,
	.ndo_uninit		= netkit_uninit,
	.ndo_bpf		= netkit_xsk,
	.ndo_xsk_wakeup		= netkit_xsk_wakeup,
	.ndo_features_check	= passthru_features_check,
};

static void netkit_get_drvinfo(struct net_device *dev,
			       struct ethtool_drvinfo *info)
{
	strscpy(info->driver, NETKIT_DRV_NAME, sizeof(info->driver));
}

static const struct ethtool_ops netkit_ethtool_ops = {
	.get_drvinfo		= netkit_get_drvinfo,
};

static int netkit_queue_create(struct net_device *dev,
			       struct netlink_ext_ack *extack)
{
	struct netkit *nk = netkit_priv(dev);
	u32 rxq_count_old, rxq_count_new;
	int err;

	rxq_count_old = dev->real_num_rx_queues;
	rxq_count_new = rxq_count_old + 1;

	/* In paired mode, only the non-primary (peer) device can
	 * create leased queues since the primary is the management
	 * side. In single device mode, leasing is always allowed.
	 */
	if (nk->pair == NETKIT_DEVICE_PAIR && nk->primary) {
		NL_SET_ERR_MSG(extack,
			       "netkit can only lease against the peer device");
		return -EOPNOTSUPP;
	}

	err = netif_set_real_num_rx_queues(dev, rxq_count_new);
	if (err) {
		if (rxq_count_new > dev->num_rx_queues)
			NL_SET_ERR_MSG(extack,
				       "netkit maximum queue limit reached");
		else
			NL_SET_ERR_MSG_FMT(extack,
					   "netkit cannot create more queues err=%d", err);
		return err;
	}

	return rxq_count_old;
}

static const struct netdev_queue_mgmt_ops netkit_queue_mgmt_ops = {
	.ndo_queue_create	= netkit_queue_create,
};

static struct net_device *netkit_alloc(struct nlattr *tb[],
				       const char *ifname,
				       unsigned char name_assign_type,
				       unsigned int num_tx_queues,
				       unsigned int num_rx_queues)
{
	const struct rtnl_link_ops *ops = &netkit_link_ops;
	struct net_device *dev;

	if (num_tx_queues > NETKIT_NUM_TX_QUEUES_MAX ||
	    num_rx_queues > NETKIT_NUM_RX_QUEUES_MAX)
		return ERR_PTR(-EOPNOTSUPP);

	dev = alloc_netdev_mqs(ops->priv_size, ifname,
			       name_assign_type, ops->setup,
			       num_tx_queues, num_rx_queues);
	if (dev) {
		dev->real_num_tx_queues = NETKIT_NUM_TX_QUEUES_REAL;
		dev->real_num_rx_queues = NETKIT_NUM_RX_QUEUES_REAL;
	}
	return dev;
}

static void netkit_queue_unlease(struct net_device *dev)
{
	struct netdev_rx_queue *rxq, *rxq_lease;
	struct net_device *dev_lease;
	int i;

	if (dev->real_num_rx_queues == 1)
		return;

	netdev_lock(dev);
	for (i = 1; i < dev->real_num_rx_queues; i++) {
		rxq = __netif_get_rx_queue(dev, i);
		rxq_lease = rxq->lease;
		dev_lease = rxq_lease->dev;

		netdev_lock(dev_lease);
		netdev_rx_queue_unlease(rxq, rxq_lease);
		netdev_unlock(dev_lease);
	}
	netdev_unlock(dev);
}

static void netkit_setup(struct net_device *dev)
{
	static const netdev_features_t netkit_features_hw_vlan =
		NETIF_F_HW_VLAN_CTAG_TX |
		NETIF_F_HW_VLAN_CTAG_RX |
		NETIF_F_HW_VLAN_STAG_TX |
		NETIF_F_HW_VLAN_STAG_RX;
	static const netdev_features_t netkit_features =
		netkit_features_hw_vlan |
		NETIF_F_SG |
		NETIF_F_FRAGLIST |
		NETIF_F_HW_CSUM |
		NETIF_F_RXCSUM |
		NETIF_F_SCTP_CRC |
		NETIF_F_HIGHDMA |
		NETIF_F_GSO_SOFTWARE |
		NETIF_F_GSO_ENCAP_ALL;

	ether_setup(dev);
	dev->max_mtu = ETH_MAX_MTU;
	dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

	dev->flags |= IFF_NOARP;
	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
	dev->priv_flags |= IFF_PHONY_HEADROOM;
	dev->priv_flags |= IFF_NO_QUEUE;
	dev->priv_flags |= IFF_DISABLE_NETPOLL;
	dev->lltx = true;

	dev->netdev_ops     = &netkit_netdev_ops;
	dev->ethtool_ops    = &netkit_ethtool_ops;
	dev->queue_mgmt_ops = &netkit_queue_mgmt_ops;

	dev->features |= netkit_features;
	dev->hw_features = netkit_features;
	dev->hw_enc_features = netkit_features;
	dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
	dev->vlan_features = dev->features & ~netkit_features_hw_vlan;

	dev->needs_free_netdev = true;

	netif_set_tso_max_size(dev, GSO_MAX_SIZE);
}

static struct net *netkit_get_link_net(const struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer = rtnl_dereference(nk->peer);

	return peer ? dev_net(peer) : dev_net(dev);
}

static int netkit_check_policy(int policy, struct nlattr *tb,
			       struct netlink_ext_ack *extack)
{
	switch (policy) {
	case NETKIT_PASS:
	case NETKIT_DROP:
		return 0;
	default:
		NL_SET_ERR_MSG_ATTR(extack, tb,
				    "Provided default xmit policy not supported");
		return -EINVAL;
	}
}

static int netkit_validate(struct nlattr *tb[], struct nlattr *data[],
			   struct netlink_ext_ack *extack)
{
	struct nlattr *attr = tb[IFLA_ADDRESS];

	if (!attr)
		return 0;
	if (nla_len(attr) != ETH_ALEN)
		return -EINVAL;
	if (!is_valid_ether_addr(nla_data(attr)))
		return -EADDRNOTAVAIL;
	return 0;
}

static int netkit_new_link(struct net_device *dev,
			   struct rtnl_newlink_params *params,
			   struct netlink_ext_ack *extack)
{
	struct net *peer_net = rtnl_newlink_peer_net(params);
	enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT;
	enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT;
	struct nlattr *peer_tb[IFLA_MAX + 1], **tbp, *attr;
	enum netkit_pairing pair = NETKIT_DEVICE_PAIR;
	enum netkit_action policy_prim = NETKIT_PASS;
	enum netkit_action policy_peer = NETKIT_PASS;
	bool seen_peer = false, seen_scrub = false;
	struct nlattr **data = params->data;
	enum netkit_mode mode = NETKIT_L3;
	unsigned char ifname_assign_type;
	struct nlattr **tb = params->tb;
	u16 headroom = 0, tailroom = 0;
	struct ifinfomsg *ifmp = NULL;
	struct net_device *peer = NULL;
	char ifname[IFNAMSIZ];
	struct netkit *nk;
	int err;

	tbp = tb;
	if (data) {
		if (data[IFLA_NETKIT_MODE])
			mode = nla_get_u32(data[IFLA_NETKIT_MODE]);
		if (data[IFLA_NETKIT_PEER_INFO]) {
			attr = data[IFLA_NETKIT_PEER_INFO];
			ifmp = nla_data(attr);
			rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack);
			tbp = peer_tb;
		}
		if (data[IFLA_NETKIT_SCRUB])
			scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]);
		if (data[IFLA_NETKIT_PEER_SCRUB])
			scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]);
		if (data[IFLA_NETKIT_POLICY]) {
			attr = data[IFLA_NETKIT_POLICY];
			policy_prim = nla_get_u32(attr);
			err = netkit_check_policy(policy_prim, attr, extack);
			if (err < 0)
				return err;
		}
		if (data[IFLA_NETKIT_PEER_POLICY]) {
			attr = data[IFLA_NETKIT_PEER_POLICY];
			policy_peer = nla_get_u32(attr);
			err = netkit_check_policy(policy_peer, attr, extack);
			if (err < 0)
				return err;
		}
		if (data[IFLA_NETKIT_HEADROOM])
			headroom = nla_get_u16(data[IFLA_NETKIT_HEADROOM]);
		if (data[IFLA_NETKIT_TAILROOM])
			tailroom = nla_get_u16(data[IFLA_NETKIT_TAILROOM]);
		if (data[IFLA_NETKIT_PAIRING])
			pair = nla_get_u32(data[IFLA_NETKIT_PAIRING]);

		seen_scrub = data[IFLA_NETKIT_SCRUB];
		seen_peer = data[IFLA_NETKIT_PEER_INFO] ||
			    data[IFLA_NETKIT_PEER_SCRUB] ||
			    data[IFLA_NETKIT_PEER_POLICY];
	}

	if (ifmp && tbp[IFLA_IFNAME]) {
		nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
		ifname_assign_type = NET_NAME_USER;
	} else {
		strscpy(ifname, "nk%d", IFNAMSIZ);
		ifname_assign_type = NET_NAME_ENUM;
	}
	if (mode != NETKIT_L2 &&
	    (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS]))
		return -EOPNOTSUPP;
	if (pair == NETKIT_DEVICE_SINGLE &&
	    (tb != tbp || seen_peer || seen_scrub ||
	     policy_prim != NETKIT_PASS))
		return -EOPNOTSUPP;

	if (pair == NETKIT_DEVICE_PAIR) {
		peer = rtnl_create_link(peer_net, ifname, ifname_assign_type,
					&netkit_link_ops, tbp, extack);
		if (IS_ERR(peer))
			return PTR_ERR(peer);

		netif_inherit_tso_max(peer, dev);
		if (headroom)
			peer->needed_headroom = headroom;
		if (tailroom)
			peer->needed_tailroom = tailroom;
		if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS]))
			eth_hw_addr_random(peer);
		if (ifmp && dev->ifindex)
			peer->ifindex = ifmp->ifi_index;

		nk = netkit_priv(peer);
		nk->primary = false;
		nk->policy = policy_peer;
		nk->scrub = scrub_peer;
		nk->mode = mode;
		nk->pair = pair;
		nk->headroom = headroom;
		bpf_mprog_bundle_init(&nk->bundle);

		err = register_netdevice(peer);
		if (err < 0)
			goto err_register_peer;
		netif_carrier_off(peer);
		if (mode == NETKIT_L2)
			dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL);

		err = rtnl_configure_link(peer, NULL, 0, NULL);
		if (err < 0)
			goto err_configure_peer;
	}

	if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS])
		eth_hw_addr_random(dev);
	if (tb[IFLA_IFNAME])
		nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
	else
		strscpy(dev->name, "nk%d", IFNAMSIZ);
	if (headroom)
		dev->needed_headroom = headroom;
	if (tailroom)
		dev->needed_tailroom = tailroom;

	nk = netkit_priv(dev);
	nk->primary = true;
	nk->policy = policy_prim;
	nk->scrub = scrub_prim;
	nk->mode = mode;
	nk->pair = pair;
	nk->headroom = headroom;
	bpf_mprog_bundle_init(&nk->bundle);

	if (pair == NETKIT_DEVICE_SINGLE)
		xdp_set_features_flag(dev, NETDEV_XDP_ACT_XSK);

	err = register_netdevice(dev);
	if (err < 0)
		goto err_configure_peer;
	netif_carrier_off(dev);
	if (mode == NETKIT_L2)
		dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL);

	rcu_assign_pointer(netkit_priv(dev)->peer, peer);
	if (peer)
		rcu_assign_pointer(netkit_priv(peer)->peer, dev);
	return 0;
err_configure_peer:
	if (peer)
		unregister_netdevice(peer);
	return err;
err_register_peer:
	free_netdev(peer);
	return err;
}

static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev,
						  bool bundle_fallback)
{
	struct netkit *nk = netkit_priv(dev);
	struct bpf_mprog_entry *entry;

	ASSERT_RTNL();
	entry = rcu_dereference_rtnl(nk->active);
	if (entry)
		return entry;
	if (bundle_fallback)
		return &nk->bundle.a;
	return NULL;
}

static void netkit_entry_update(struct net_device *dev,
				struct bpf_mprog_entry *entry)
{
	struct netkit *nk = netkit_priv(dev);

	ASSERT_RTNL();
	rcu_assign_pointer(nk->active, entry);
}

static void netkit_entry_sync(void)
{
	synchronize_rcu();
}

static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which)
{
	struct net_device *dev;
	struct netkit *nk;

	ASSERT_RTNL();

	switch (which) {
	case BPF_NETKIT_PRIMARY:
	case BPF_NETKIT_PEER:
		break;
	default:
		return ERR_PTR(-EINVAL);
	}

	dev = __dev_get_by_index(net, ifindex);
	if (!dev)
		return ERR_PTR(-ENODEV);
	if (dev->netdev_ops != &netkit_netdev_ops)
		return ERR_PTR(-ENXIO);

	nk = netkit_priv(dev);
	if (!nk->primary)
		return ERR_PTR(-EACCES);
	if (nk->pair == NETKIT_DEVICE_SINGLE)
		return ERR_PTR(-EOPNOTSUPP);
	if (which == BPF_NETKIT_PEER) {
		dev = rcu_dereference_rtnl(nk->peer);
		if (!dev)
			return ERR_PTR(-ENODEV);
	}
	return dev;
}

int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
{
	struct bpf_mprog_entry *entry, *entry_new;
	struct bpf_prog *replace_prog = NULL;
	struct net_device *dev;
	int ret;

	rtnl_lock();
	dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
			       attr->attach_type);
	if (IS_ERR(dev)) {
		ret = PTR_ERR(dev);
		goto out;
	}
	entry = netkit_entry_fetch(dev, true);
	if (attr->attach_flags & BPF_F_REPLACE) {
		replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
						 prog->type);
		if (IS_ERR(replace_prog)) {
			ret = PTR_ERR(replace_prog);
			replace_prog = NULL;
			goto out;
		}
	}
	ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
			       attr->attach_flags, attr->relative_fd,
			       attr->expected_revision);
	if (!ret) {
		if (entry != entry_new) {
			netkit_entry_update(dev, entry_new);
			netkit_entry_sync();
		}
		bpf_mprog_commit(entry);
	}
out:
	if (replace_prog)
		bpf_prog_put(replace_prog);
	rtnl_unlock();
	return ret;
}

int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
{
	struct bpf_mprog_entry *entry, *entry_new;
	struct net_device *dev;
	int ret;

	rtnl_lock();
	dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
			       attr->attach_type);
	if (IS_ERR(dev)) {
		ret = PTR_ERR(dev);
		goto out;
	}
	entry = netkit_entry_fetch(dev, false);
	if (!entry) {
		ret = -ENOENT;
		goto out;
	}
	ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
			       attr->relative_fd, attr->expected_revision);
	if (!ret) {
		if (!bpf_mprog_total(entry_new))
			entry_new = NULL;
		netkit_entry_update(dev, entry_new);
		netkit_entry_sync();
		bpf_mprog_commit(entry);
	}
out:
	rtnl_unlock();
	return ret;
}

int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
{
	struct net_device *dev;
	int ret;

	rtnl_lock();
	dev = netkit_dev_fetch(current->nsproxy->net_ns,
			       attr->query.target_ifindex,
			       attr->query.attach_type);
	if (IS_ERR(dev)) {
		ret = PTR_ERR(dev);
		goto out;
	}
	ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false));
out:
	rtnl_unlock();
	return ret;
}

static struct netkit_link *netkit_link(const struct bpf_link *link)
{
	return container_of(link, struct netkit_link, link);
}

static int netkit_link_prog_attach(struct bpf_link *link, u32 flags,
				   u32 id_or_fd, u64 revision)
{
	struct netkit_link *nkl = netkit_link(link);
	struct bpf_mprog_entry *entry, *entry_new;
	struct net_device *dev = nkl->dev;
	int ret;

	ASSERT_RTNL();
	entry = netkit_entry_fetch(dev, true);
	ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
			       id_or_fd, revision);
	if (!ret) {
		if (entry != entry_new) {
			netkit_entry_update(dev, entry_new);
			netkit_entry_sync();
		}
		bpf_mprog_commit(entry);
	}
	return ret;
}

static void netkit_link_release(struct bpf_link *link)
{
	struct netkit_link *nkl = netkit_link(link);
	struct bpf_mprog_entry *entry, *entry_new;
	struct net_device *dev;
	int ret = 0;

	rtnl_lock();
	dev = nkl->dev;
	if (!dev)
		goto out;
	entry = netkit_entry_fetch(dev, false);
	if (!entry) {
		ret = -ENOENT;
		goto out;
	}
	ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
	if (!ret) {
		if (!bpf_mprog_total(entry_new))
			entry_new = NULL;
		netkit_entry_update(dev, entry_new);
		netkit_entry_sync();
		bpf_mprog_commit(entry);
		nkl->dev = NULL;
	}
out:
	WARN_ON_ONCE(ret);
	rtnl_unlock();
}

static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog,
			      struct bpf_prog *oprog)
{
	struct netkit_link *nkl = netkit_link(link);
	struct bpf_mprog_entry *entry, *entry_new;
	struct net_device *dev;
	int ret = 0;

	rtnl_lock();
	dev = nkl->dev;
	if (!dev) {
		ret = -ENOLINK;
		goto out;
	}
	if (oprog && link->prog != oprog) {
		ret = -EPERM;
		goto out;
	}
	oprog = link->prog;
	if (oprog == nprog) {
		bpf_prog_put(nprog);
		goto out;
	}
	entry = netkit_entry_fetch(dev, false);
	if (!entry) {
		ret = -ENOENT;
		goto out;
	}
	ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
			       BPF_F_REPLACE | BPF_F_ID,
			       link->prog->aux->id, 0);
	if (!ret) {
		WARN_ON_ONCE(entry != entry_new);
		oprog = xchg(&link->prog, nprog);
		bpf_prog_put(oprog);
		bpf_mprog_commit(entry);
	}
out:
	rtnl_unlock();
	return ret;
}

static void netkit_link_dealloc(struct bpf_link *link)
{
	kfree(netkit_link(link));
}

static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
{
	const struct netkit_link *nkl = netkit_link(link);
	u32 ifindex = 0;

	rtnl_lock();
	if (nkl->dev)
		ifindex = nkl->dev->ifindex;
	rtnl_unlock();

	seq_printf(seq, "ifindex:\t%u\n", ifindex);
	seq_printf(seq, "attach_type:\t%u (%s)\n",
		   link->attach_type,
		   link->attach_type == BPF_NETKIT_PRIMARY ? "primary" : "peer");
}

static int netkit_link_fill_info(const struct bpf_link *link,
				 struct bpf_link_info *info)
{
	const struct netkit_link *nkl = netkit_link(link);
	u32 ifindex = 0;

	rtnl_lock();
	if (nkl->dev)
		ifindex = nkl->dev->ifindex;
	rtnl_unlock();

	info->netkit.ifindex = ifindex;
	info->netkit.attach_type = link->attach_type;
	return 0;
}

static int netkit_link_detach(struct bpf_link *link)
{
	netkit_link_release(link);
	return 0;
}

static const struct bpf_link_ops netkit_link_lops = {
	.release	= netkit_link_release,
	.detach		= netkit_link_detach,
	.dealloc	= netkit_link_dealloc,
	.update_prog	= netkit_link_update,
	.show_fdinfo	= netkit_link_fdinfo,
	.fill_link_info	= netkit_link_fill_info,
};

static int netkit_link_init(struct netkit_link *nkl,
			    struct bpf_link_primer *link_primer,
			    const union bpf_attr *attr,
			    struct net_device *dev,
			    struct bpf_prog *prog)
{
	bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT,
		      &netkit_link_lops, prog, attr->link_create.attach_type);
	nkl->dev = dev;
	return bpf_link_prime(&nkl->link, link_primer);
}

int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
{
	struct bpf_link_primer link_primer;
	struct netkit_link *nkl;
	struct net_device *dev;
	int ret;

	rtnl_lock();
	dev = netkit_dev_fetch(current->nsproxy->net_ns,
			       attr->link_create.target_ifindex,
			       attr->link_create.attach_type);
	if (IS_ERR(dev)) {
		ret = PTR_ERR(dev);
		goto out;
	}
	nkl = kzalloc_obj(*nkl, GFP_KERNEL_ACCOUNT);
	if (!nkl) {
		ret = -ENOMEM;
		goto out;
	}
	ret = netkit_link_init(nkl, &link_primer, attr, dev, prog);
	if (ret) {
		kfree(nkl);
		goto out;
	}
	ret = netkit_link_prog_attach(&nkl->link,
				      attr->link_create.flags,
				      attr->link_create.netkit.relative_fd,
				      attr->link_create.netkit.expected_revision);
	if (ret) {
		nkl->dev = NULL;
		bpf_link_cleanup(&link_primer);
		goto out;
	}
	ret = bpf_link_settle(&link_primer);
out:
	rtnl_unlock();
	return ret;
}

static void netkit_release_all(struct net_device *dev)
{
	struct bpf_mprog_entry *entry;
	struct bpf_tuple tuple = {};
	struct bpf_mprog_fp *fp;
	struct bpf_mprog_cp *cp;

	entry = netkit_entry_fetch(dev, false);
	if (!entry)
		return;
	netkit_entry_update(dev, NULL);
	netkit_entry_sync();
	bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
		if (tuple.link)
			netkit_link(tuple.link)->dev = NULL;
		else
			bpf_prog_put(tuple.prog);
	}
}

static void netkit_uninit(struct net_device *dev)
{
	netkit_release_all(dev);
	netkit_queue_unlease(dev);
}

static void netkit_del_link(struct net_device *dev, struct list_head *head)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer = rtnl_dereference(nk->peer);

	RCU_INIT_POINTER(nk->peer, NULL);
	unregister_netdevice_queue(dev, head);
	if (peer) {
		nk = netkit_priv(peer);
		RCU_INIT_POINTER(nk->peer, NULL);
		/* Guard against the peer already being in an unregister
		 * list (e.g. same-namespace teardown where the peer is
		 * in the caller's dev_kill_list). list_move_tail() on an
		 * already-queued device would otherwise corrupt that
		 * list's iteration. This situation can occur via netkit
		 * notifier, hence guard against this scenario.
		 */
		if (!unregister_netdevice_queued(peer))
			unregister_netdevice_queue(peer, head);
	}
}

static int netkit_change_link(struct net_device *dev, struct nlattr *tb[],
			      struct nlattr *data[],
			      struct netlink_ext_ack *extack)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer = rtnl_dereference(nk->peer);
	enum netkit_action policy;
	struct nlattr *attr;
	int err, i;
	static const struct {
		u32 attr;
		char *name;
	} fixed_params[] = {
		{ IFLA_NETKIT_MODE,       "operating mode" },
		{ IFLA_NETKIT_SCRUB,      "scrubbing" },
		{ IFLA_NETKIT_PEER_SCRUB, "peer scrubbing" },
		{ IFLA_NETKIT_PEER_INFO,  "peer info" },
		{ IFLA_NETKIT_HEADROOM,   "headroom" },
		{ IFLA_NETKIT_TAILROOM,   "tailroom" },
		{ IFLA_NETKIT_PAIRING,    "pairing" },
	};

	if (!nk->primary) {
		NL_SET_ERR_MSG(extack,
			       "netkit link settings can be changed only through the primary device");
		return -EACCES;
	}

	for (i = 0; i < ARRAY_SIZE(fixed_params); i++) {
		attr = data[fixed_params[i].attr];
		if (attr) {
			NL_SET_ERR_MSG_ATTR_FMT(extack, attr,
						"netkit link %s cannot be changed after device creation",
						fixed_params[i].name);
			return -EACCES;
		}
	}

	if (data[IFLA_NETKIT_POLICY]) {
		err = -EOPNOTSUPP;
		attr = data[IFLA_NETKIT_POLICY];
		policy = nla_get_u32(attr);
		if (nk->pair == NETKIT_DEVICE_PAIR)
			err = netkit_check_policy(policy, attr, extack);
		if (err)
			return err;
		WRITE_ONCE(nk->policy, policy);
	}

	if (data[IFLA_NETKIT_PEER_POLICY]) {
		err = -EOPNOTSUPP;
		attr = data[IFLA_NETKIT_PEER_POLICY];
		policy = nla_get_u32(attr);
		if (peer)
			err = netkit_check_policy(policy, attr, extack);
		if (err)
			return err;
		nk = netkit_priv(peer);
		WRITE_ONCE(nk->policy, policy);
	}

	return 0;
}

static void netkit_check_lease_unregister(struct net_device *dev)
{
	LIST_HEAD(list_kill);
	u32 q_idx;

	if (READ_ONCE(dev->reg_state) != NETREG_UNREGISTERING ||
	    !dev->dev.parent)
		return;

	netdev_lock_ops(dev);
	for (q_idx = 0; q_idx < dev->real_num_rx_queues; q_idx++) {
		struct net_device *tmp = dev;
		struct netdev_rx_queue *rxq;
		u32 tmp_q_idx = q_idx;

		rxq = __netif_get_rx_queue_lease(&tmp, &tmp_q_idx,
						 NETIF_PHYS_TO_VIRT);
		if (rxq && tmp != dev &&
		    tmp->netdev_ops == &netkit_netdev_ops) {
			/* A single phys device can have multiple queues leased
			 * to one netkit device. We can only queue that netkit
			 * device once to the list_kill. Queues of that phys
			 * device can be leased with different individual netkit
			 * devices, hence we batch via list_kill.
			 */
			if (unregister_netdevice_queued(tmp))
				continue;
			netkit_del_link(tmp, &list_kill);
		}
	}
	netdev_unlock_ops(dev);
	unregister_netdevice_many(&list_kill);
}

static int netkit_notifier(struct notifier_block *this,
			   unsigned long event, void *ptr)
{
	struct net_device *dev = netdev_notifier_info_to_dev(ptr);

	if (event == NETDEV_UNREGISTER)
		netkit_check_lease_unregister(dev);
	return NOTIFY_DONE;
}

static size_t netkit_get_size(const struct net_device *dev)
{
	return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */
	       nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */
	       nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */
	       nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */
	       nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */
	       nla_total_size(sizeof(u8))  + /* IFLA_NETKIT_PRIMARY */
	       nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_HEADROOM */
	       nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_TAILROOM */
	       nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PAIRING */
	       0;
}

static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer = rtnl_dereference(nk->peer);

	if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary))
		return -EMSGSIZE;
	if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy))
		return -EMSGSIZE;
	if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode))
		return -EMSGSIZE;
	if (nk->pair == NETKIT_DEVICE_PAIR &&
	    nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub))
		return -EMSGSIZE;
	if (nla_put_u16(skb, IFLA_NETKIT_HEADROOM, dev->needed_headroom))
		return -EMSGSIZE;
	if (nla_put_u16(skb, IFLA_NETKIT_TAILROOM, dev->needed_tailroom))
		return -EMSGSIZE;
	if (nla_put_u32(skb, IFLA_NETKIT_PAIRING, nk->pair))
		return -EMSGSIZE;

	if (peer) {
		nk = netkit_priv(peer);
		if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy))
			return -EMSGSIZE;
		if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub))
			return -EMSGSIZE;
	}

	return 0;
}

static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = {
	[IFLA_NETKIT_PEER_INFO]		= { .len = sizeof(struct ifinfomsg) },
	[IFLA_NETKIT_MODE]		= NLA_POLICY_MAX(NLA_U32, NETKIT_L3),
	[IFLA_NETKIT_POLICY]		= { .type = NLA_U32 },
	[IFLA_NETKIT_PEER_POLICY]	= { .type = NLA_U32 },
	[IFLA_NETKIT_HEADROOM]		= { .type = NLA_U16 },
	[IFLA_NETKIT_TAILROOM]		= { .type = NLA_U16 },
	[IFLA_NETKIT_SCRUB]		= NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
	[IFLA_NETKIT_PEER_SCRUB]	= NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
	[IFLA_NETKIT_PAIRING]		= NLA_POLICY_MAX(NLA_U32, NETKIT_DEVICE_SINGLE),
	[IFLA_NETKIT_PRIMARY]		= { .type = NLA_REJECT,
					    .reject_message = "Primary attribute is read-only" },
};

static struct rtnl_link_ops netkit_link_ops = {
	.kind		= NETKIT_DRV_NAME,
	.priv_size	= sizeof(struct netkit),
	.alloc		= netkit_alloc,
	.setup		= netkit_setup,
	.newlink	= netkit_new_link,
	.dellink	= netkit_del_link,
	.changelink	= netkit_change_link,
	.get_link_net	= netkit_get_link_net,
	.get_size	= netkit_get_size,
	.fill_info	= netkit_fill_info,
	.policy		= netkit_policy,
	.validate	= netkit_validate,
	.peer_type	= IFLA_NETKIT_PEER_INFO,
	.maxtype	= IFLA_NETKIT_MAX,
};

static struct notifier_block netkit_netdev_notifier = {
	.notifier_call	= netkit_notifier,
};

static __init int netkit_mod_init(void)
{
	int ret;

	BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT ||
		     (int)NETKIT_PASS != (int)TCX_PASS ||
		     (int)NETKIT_DROP != (int)TCX_DROP ||
		     (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);

	ret = rtnl_link_register(&netkit_link_ops);
	if (ret)
		return ret;
	ret = register_netdevice_notifier(&netkit_netdev_notifier);
	if (ret)
		rtnl_link_unregister(&netkit_link_ops);
	return ret;
}

static __exit void netkit_mod_exit(void)
{
	unregister_netdevice_notifier(&netkit_netdev_notifier);
	rtnl_link_unregister(&netkit_link_ops);
}

module_init(netkit_mod_init);
module_exit(netkit_mod_exit);

MODULE_DESCRIPTION("BPF-programmable network device");
MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>");
MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_RTNL_LINK(NETKIT_DRV_NAME);