path: root/net/bpf/test_run.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2017 Facebook
 */
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/vmalloc.h>
#include <linux/etherdevice.h>
#include <linux/filter.h>
#include <linux/rcupdate_trace.h>
#include <linux/sched/signal.h>
#include <net/bpf_sk_storage.h>
#include <net/sock.h>
#include <net/tcp.h>
#include <net/net_namespace.h>
#include <net/page_pool.h>
#include <linux/error-injection.h>
#include <linux/smp.h>
#include <linux/sock_diag.h>
#include <net/xdp.h>

#define CREATE_TRACE_POINTS
#include <trace/events/bpf_test_run.h>

struct bpf_test_timer {
	enum { NO_PREEMPT, NO_MIGRATE } mode;
	u32 i;
	u64 time_start, time_spent;
};

static void bpf_test_timer_enter(struct bpf_test_timer *t)
	__acquires(rcu)
{
	rcu_read_lock();
	if (t->mode == NO_PREEMPT)
		preempt_disable();
	else
		migrate_disable();

	t->time_start = ktime_get_ns();
}

static void bpf_test_timer_leave(struct bpf_test_timer *t)
	__releases(rcu)
{
	t->time_start = 0;

	if (t->mode == NO_PREEMPT)
		preempt_enable();
	else
		migrate_enable();
	rcu_read_unlock();
}

static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,
				    u32 repeat, int *err, u32 *duration)
	__must_hold(rcu)
{
	t->i += iterations;
	if (t->i >= repeat) {
		/* We're done. */
		t->time_spent += ktime_get_ns() - t->time_start;
		do_div(t->time_spent, t->i);
		*duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
		*err = 0;
		goto reset;
	}

	if (signal_pending(current)) {
		/* During iteration: we've been cancelled, abort. */
		*err = -EINTR;
		goto reset;
	}

	if (need_resched()) {
		/* During iteration: we need to reschedule between runs. */
		t->time_spent += ktime_get_ns() - t->time_start;
		bpf_test_timer_leave(t);
		cond_resched();
		bpf_test_timer_enter(t);
	}

	/* Do another round. */
	return true;

reset:
	t->i = 0;
	return false;
}

/* We put this struct at the head of each page with a context and frame
 * initialised when the page is allocated, so we don't have to do this on each
 * repetition of the test run.
 */
struct xdp_page_head {
	struct xdp_buff orig_ctx;
	struct xdp_buff ctx;
	struct xdp_frame frm;
	u8 data[];
};

struct xdp_test_data {
	struct xdp_buff *orig_ctx;
	struct xdp_rxq_info rxq;
	struct net_device *dev;
	struct page_pool *pp;
	struct xdp_frame **frames;
	struct sk_buff **skbs;
	struct xdp_mem_info mem;
	u32 batch_size;
	u32 frame_cnt;
};

#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
#define TEST_XDP_MAX_BATCH 256

static void xdp_test_run_init_page(struct page *page, void *arg)
{
	struct xdp_page_head *head = phys_to_virt(page_to_phys(page));
	struct xdp_buff *new_ctx, *orig_ctx;
	u32 headroom = XDP_PACKET_HEADROOM;
	struct xdp_test_data *xdp = arg;
	size_t frm_len, meta_len;
	struct xdp_frame *frm;
	void *data;

	orig_ctx = xdp->orig_ctx;
	frm_len = orig_ctx->data_end - orig_ctx->data_meta;
	meta_len = orig_ctx->data - orig_ctx->data_meta;
	headroom -= meta_len;

	new_ctx = &head->ctx;
	frm = &head->frm;
	data = &head->data;
	memcpy(data + headroom, orig_ctx->data_meta, frm_len);

	xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);
	xdp_prepare_buff(new_ctx, data, headroom, frm_len, true);
	new_ctx->data = new_ctx->data_meta + meta_len;

	xdp_update_frame_from_buff(new_ctx, frm);
	frm->mem = new_ctx->rxq->mem;

	memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx));
}

static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)
{
	struct page_pool *pp;
	int err = -ENOMEM;
	struct page_pool_params pp_params = {
		.order = 0,
		.flags = 0,
		.pool_size = xdp->batch_size,
		.nid = NUMA_NO_NODE,
		.init_callback = xdp_test_run_init_page,
		.init_arg = xdp,
	};

	xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
	if (!xdp->frames)
		return -ENOMEM;

	xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *)