path: root/drivers/net/netkit.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2023 Isovalent */

#include <linux/netdevice.h>
#include <linux/ethtool.h>
#include <linux/etherdevice.h>
#include <linux/filter.h>
#include <linux/netfilter_netdev.h>
#include <linux/bpf_mprog.h>
#include <linux/indirect_call_wrapper.h>

#include <net/netkit.h>
#include <net/dst.h>
#include <net/tcx.h>

#define DRV_NAME "netkit"

struct netkit {
	/* Needed in fast-path */
	struct net_device __rcu *peer;
	struct bpf_mprog_entry __rcu *active;
	enum netkit_action policy;
	struct bpf_mprog_bundle	bundle;

	/* Needed in slow-path */
	enum netkit_mode mode;
	bool primary;
	u32 headroom;
};

struct netkit_link {
	struct bpf_link link;
	struct net_device *dev;
	u32 location;
};

static __always_inline int
netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
	   enum netkit_action ret)
{
	const struct bpf_mprog_fp *fp;
	const struct bpf_prog *prog;

	bpf_mprog_foreach_prog(entry, fp, prog) {
		bpf_compute_data_pointers(skb);
		ret = bpf_prog_run(prog, skb);
		if (ret != NETKIT_NEXT)
			break;
	}
	return ret;
}

static void netkit_prep_forward(struct sk_buff *skb, bool xnet)
{
	skb_scrub_packet(skb, xnet);
	skb->priority = 0;
	nf_skip_egress(skb, true);
}

static struct netkit *netkit_priv(const struct net_device *dev)
{
	return netdev_priv(dev);
}

static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	enum netkit_action ret = READ_ONCE(nk->policy);
	netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
	const struct bpf_mprog_entry *entry;
	struct net_device *peer;
	int len = skb->len;

	rcu_read_lock();
	peer = rcu_dereference(nk->peer);
	if (unlikely(!peer || !(peer->flags & IFF_UP) ||
		     !pskb_may_pull(skb, ETH_HLEN) ||
		     skb_orphan_frags(skb, GFP_ATOMIC)))
		goto drop;
	netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)));
	skb->dev = peer;
	entry = rcu_dereference(nk->active);
	if (entry)
		ret = netkit_run(entry, skb, ret);
	switch (ret) {
	case NETKIT_NEXT:
	case NETKIT_PASS:
		skb->protocol = eth_type_trans(skb, skb->dev);
		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
		if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
			dev_sw_netstats_tx_add(dev, 1, len);
			dev_sw_netstats_rx_add(peer, len);
		} else {
			goto drop_stats;
		}
		break;
	case NETKIT_REDIRECT:
		dev_sw_netstats_tx_add(dev, 1, len);
		skb_do_redirect(skb);
		break;
	case NETKIT_DROP:
	default:
drop:
		kfree_skb(skb);
drop_stats:
		dev_core_stats_tx_dropped_inc(dev);
		ret_dev = NET_XMIT_DROP;
		break;
	}
	rcu_read_unlock();
	return ret_dev;
}

static int netkit_open(struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer = rtnl_dereference(nk->peer);

	if (!peer)
		return -ENOTCONN;
	if (peer->flags & IFF_UP) {
		netif_carrier_on(dev);
		netif_carrier_on(peer);
	}
	return 0;
}

static int netkit_close(struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer = rtnl_dereference(nk->peer);

	netif_carrier_off(dev);
	if (peer)
		netif_carrier_off(peer);
	return 0;
}

static int netkit_get_iflink(const struct net_device *dev)
{
	struct netkit *nk = netkit_priv(dev);
	struct net_device *peer;
	int iflink = 0;

	rcu_read_lock();
	peer = rcu_dereference(nk->peer);
	if (peer)
		iflink = peer->ifindex;
	rcu_read_unlock();
	return iflink;
}

static void netkit_set_multicast(struct net_device *dev)
{
	/* Nothing to do, we receive whatever gets pushed to us! */
}

static void netkit_set_headroom(struct net_device *dev, int headroom)
{
	struct netkit *nk = netkit_priv(dev), *nk2;
	struct net_device *peer;

	if (headroom < 0)
		headroom = NET_SKB_PAD;

	rcu_read_lock();
	peer = rcu_dereference(nk->peer);
	if (unlikely(!peer))
		goto out;

	nk2 = netkit_priv(peer);
	nk->headroom = headroom;
	headroom = max(nk->headroom, nk2->headroom);

	peer->needed_headroom = headroom;
	dev->needed_headroom = headroom;
out:
	rcu_read_unlock();
}

INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
{
	return rcu_dereference(netkit_priv(dev)->peer);
}

static void netkit_get_stats(struct net_device *dev,
			     struct rtnl_link_stats64 *stats)
{
	dev_fetch_sw_netstats(stats, dev->tstats);
	stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
}

static void netkit_uninit(struct net_device *dev);

static const struct net_device_ops netkit_netdev_ops = {
	.ndo_open		= netkit_open,
	.ndo_stop		= netkit_close,
	.ndo_start_xmit		= netkit_xmit,
	.ndo_set_rx_mode	= netkit_set_multicast,
	.ndo_set_rx_headroom	= netkit_set_headroom,
	.ndo_get_iflink		= netkit_get_iflink,
	.ndo_get_pe