path: root/net/bpf/test_run.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2017 Facebook
 */
#include <linux/bpf.h>
#include <linux/btf_ids.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/etherdevice.h>
#include <linux/filter.h>
#include <linux/rcupdate_trace.h>
#include <linux/sched/signal.h>
#include <net/bpf_sk_storage.h>
#include <net/sock.h>
#include <net/tcp.h>
#include <net/net_namespace.h>
#include <linux/error-injection.h>
#include <linux/smp.h>
#include <linux/sock_diag.h>
#include <net/xdp.h>

#define CREATE_TRACE_POINTS
#include <trace/events/bpf_test_run.h>

struct bpf_test_timer {
	enum { NO_PREEMPT, NO_MIGRATE } mode;
	u32 i;
	u64 time_start, time_spent;
};

static void bpf_test_timer_enter(struct bpf_test_timer *t)
	__acquires(rcu)
{
	rcu_read_lock();
	if (t->mode == NO_PREEMPT)
		preempt_disable();
	else
		migrate_disable();

	t->time_start = ktime_get_ns();
}

static void bpf_test_timer_leave(struct bpf_test_timer *t)
	__releases(rcu)
{
	t->time_start = 0;

	if (t->mode == NO_PREEMPT)
		preempt_enable();
	else
		migrate_enable();
	rcu_read_unlock();
}

static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration)
	__must_hold(rcu)
{
	t->i++;
	if (t->i >= repeat) {
		/* We're done. */
		t->time_spent += ktime_get_ns() - t->time_start;
		do_div(t->time_spent, t->i);
		*duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
		*err = 0;
		goto reset;
	}

	if (signal_pending(current)) {
		/* During iteration: we've been cancelled, abort. */
		*err = -EINTR;
		goto reset;
	}

	if (need_resched()) {
		/* During iteration: we need to reschedule between runs. */
		t->time_spent += ktime_get_ns() - t->time_start;
		bpf_test_timer_leave(t);
		cond_resched();
		bpf_test_timer_enter(t);
	}

	/* Do another round. */
	return true;

reset:
	t->i = 0;
	return false;
}

static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
			u32 *retval, u32 *time, bool xdp)
{
	struct bpf_prog_array_item item = {.prog = prog};
	struct bpf_run_ctx *old_ctx;
	struct bpf_cg_run_ctx run_ctx;
	struct bpf_test_timer t = { NO_MIGRATE };
	enum bpf_cgroup_storage_type stype;
	int ret;

	for_each_cgroup_storage_type(stype) {
		item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
		if (IS_ERR(item.cgroup_storage[stype])) {
			item.cgroup_storage[stype] = NULL;
			for_each_cgroup_storage_type(stype)
				bpf_cgroup_storage_free(item.cgroup_storage[stype]);
			return -ENOMEM;
		}
	}

	if (!repeat)
		repeat = 1;

	bpf_test_timer_enter(&t);
	old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
	do {
		run_ctx.prog_item = &item;
		if (xdp)
			*retval = bpf_prog_run_xdp(prog, ctx);
		else
			*retval = bpf_prog_run(prog, ctx);
	} while (bpf_test_timer_continue(&t, repeat, &ret, time));
	bpf_reset_run_ctx(old_ctx);
	bpf_test_timer_leave(&t);

	for_each_cgroup_storage_type(stype)
		bpf_cgroup_storage_free(item.cgroup_storage[stype]);

	return ret;
}

static int bpf_test_finish(const union bpf_attr *kattr,
			   union bpf_attr __user *uattr, const void *data,
			   u32 size, u32 retval, u32 duration)
{
	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
	int err = -EFAULT;
	u32 copy_size = size;

	/* Clamp copy if the user has provided a size hint, but copy the full
	 * buffer if not to retain old behaviour.
	 */
	if (kattr->test.data_size_out &&
	    copy_size > kattr->test.data_size_out) {
		copy_size = kattr->test.data_size_out;
		err = -ENOSPC;
	}

	if (data_out && copy_to_user(data_out, data, copy_size))
		goto out;
	if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
		goto out;
	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
		goto out;
	if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
		goto out;
	if (err != -ENOSPC)
		err = 0;
out:
	trace_bpf_test_finish(&err);
	return err;
}

/* Integer types of various sizes and pointer combinations cover va