path: root/kernel/bpf/trampoline.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2019 Facebook */
#include <linux/hash.h>
#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/ftrace.h>
#include <linux/rbtree_latch.h>
#include <linux/perf_event.h>
#include <linux/btf.h>
#include <linux/rcupdate_trace.h>
#include <linux/rcupdate_wait.h>
#include <linux/static_call.h>
#include <linux/bpf_verifier.h>
#include <linux/bpf_lsm.h>
#include <linux/delay.h>

/* dummy _ops. The verifier will operate on target program's ops. */
const struct bpf_verifier_ops bpf_extension_verifier_ops = {
};
const struct bpf_prog_ops bpf_extension_prog_ops = {
};

/* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */
#define TRAMPOLINE_HASH_BITS 10
#define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS)

static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];

/* serializes access to trampoline_table */
static DEFINE_MUTEX(trampoline_mutex);

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);

static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd)
{
	struct bpf_trampoline *tr = ops->private;
	int ret = 0;

	if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) {
		/* This is called inside register_ftrace_direct_multi(), so
		 * tr->mutex is already locked.
		 */
		lockdep_assert_held_once(&tr->mutex);

		/* Instead of updating the trampoline here, we propagate
		 * -EAGAIN to register_ftrace_direct(). Then we can
		 * retry register_ftrace_direct() after updating the
		 * trampoline.
		 */
		if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
		    !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) {
			if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY))
				return -EBUSY;

			tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;
			return -EAGAIN;
		}

		return 0;
	}

	/* The normal locking order is
	 *    tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
	 *
	 * The following two commands are called from
	 *
	 *   prepare_direct_functions_for_ipmodify
	 *   cleanup_direct_functions_after_ipmodify
	 *
	 * In both cases, direct_mutex is already locked. Use
	 * mutex_trylock(&tr->mutex) to avoid deadlock in race condition
	 * (something else is making changes to this same trampoline).
	 */
	if (!mutex_trylock(&tr->mutex)) {
		/* sleep 1 ms to make sure whatever holding tr->mutex makes
		 * some progress.
		 */
		msleep(1);
		return -EAGAIN;
	}

	switch (cmd) {
	case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER:
		tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY;

		if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
		    !(tr->flags & BPF_TRAMP_F_ORIG_STACK))
			ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
		break;
	case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER:
		tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY;

		if (tr->flags & BPF_TRAMP_F_ORIG_STACK)
			ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
		break;
	default:
		ret = -EINVAL;
		break;
	}

	mutex_unlock(&tr->mutex);
	return ret;
}
#endif

bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
{
	enum bpf_attach_type eatype = prog->expected_attach_type;
	enum bpf_prog_type ptype = prog->type;

	return (ptype == BPF_PROG_TYPE_TRACING &&
		(eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
		 eatype == BPF_MODIFY_RETURN)) ||
		(ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
}

void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym)
{
	ksym->start = (unsigned long) data;
	ksym->end = ksym->start + size;
	bpf_ksym_add(ksym);
	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
			   PAGE_SIZE, false, ksym->name);
}

void bpf_image_ksym_del(struct bpf_ksym *ksym)
{
	bpf_ksym_del(ksym);
	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
			   PAGE_SIZE, true, ksym->name);
}

static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
{
	struct bpf_trampoline *tr;
	struct hlist_head *head;
	int i;

	mutex_lock(&trampoline_mutex);
	head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
	hlist_for_each_entry(tr, head, hlist) {
		if (tr->key == key) {
			refcount_inc(&tr->refcnt);
			goto out;
		}
	}
	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
	if (!tr)
		goto out;
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
	tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
	if (!tr->fops) {
		kfree(