path: root/kernel/bpf/bpf_struct_ops.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2019 Facebook */

#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/btf.h>
#include <linux/filter.h>
#include <linux/slab.h>
#include <linux/numa.h>
#include <linux/seq_file.h>
#include <linux/refcount.h>
#include <linux/mutex.h>
#include <linux/btf_ids.h>
#include <linux/rcupdate_wait.h>

struct bpf_struct_ops_value {
	struct bpf_struct_ops_common_value common;
	char data[] ____cacheline_aligned_in_smp;
};

#define MAX_TRAMP_IMAGE_PAGES 8

struct bpf_struct_ops_map {
	struct bpf_map map;
	struct rcu_head rcu;
	const struct bpf_struct_ops_desc *st_ops_desc;
	/* protect map_update */
	struct mutex lock;
	/* link has all the bpf_links that is populated
	 * to the func ptr of the kernel's struct
	 * (in kvalue.data).
	 */
	struct bpf_link **links;
	u32 links_cnt;
	u32 image_pages_cnt;
	/* image_pages is an array of pages that has all the trampolines
	 * that stores the func args before calling the bpf_prog.
	 */
	void *image_pages[MAX_TRAMP_IMAGE_PAGES];
	/* The owner moduler's btf. */
	struct btf *btf;
	/* uvalue->data stores the kernel struct
	 * (e.g. tcp_congestion_ops) that is more useful
	 * to userspace than the kvalue.  For example,
	 * the bpf_prog's id is stored instead of the kernel
	 * address of a func ptr.
	 */
	struct bpf_struct_ops_value *uvalue;
	/* kvalue.data stores the actual kernel's struct
	 * (e.g. tcp_congestion_ops) that will be
	 * registered to the kernel subsystem.
	 */
	struct bpf_struct_ops_value kvalue;
};

struct bpf_struct_ops_link {
	struct bpf_link link;
	struct bpf_map __rcu *map;
};

static DEFINE_MUTEX(update_mutex);

#define VALUE_PREFIX "bpf_struct_ops_"
#define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)

const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
};

const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
#ifdef CONFIG_NET
	.test_run = bpf_struct_ops_test_run,
#endif
};

BTF_ID_LIST(st_ops_ids)
BTF_ID(struct, module)
BTF_ID(struct, bpf_struct_ops_common_value)

enum {
	IDX_MODULE_ID,
	IDX_ST_OPS_COMMON_VALUE_ID,
};

extern struct btf *btf_vmlinux;

static bool is_valid_value_type(struct btf *btf, s32 value_id,
				const struct btf_type *type,
				const char *value_name)
{
	const struct btf_type *common_value_type;
	const struct btf_member *member;
	const struct btf_type *vt, *mt;

	vt = btf_type_by_id(btf, value_id);
	if (btf_vlen(vt) != 2) {
		pr_warn("The number of %s's members should be 2, but we get %d\n",
			value_name, btf_vlen(vt));
		return false;
	}
	member = btf_type_member(vt);
	mt = btf_type_by_id(btf, member->type);
	common_value_type = btf_type_by_id(btf_vmlinux,
					   st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]);
	if (mt != common_value_type) {
		pr_warn("The first member of %s should be bpf_struct_ops_common_value\n",
			value_name);
		return false;
	}
	member++;
	mt = btf_type_by_id(btf, member->type);
	if (mt != type) {
		pr_warn("The second member of %s should be %s\n",
			value_name, btf_name_by_offset(btf, type->name_off));
		return false;
	}

	return true;
}

static void *bpf_struct_ops_image_alloc(void)
{
	void *image;
	int err;

	err = bpf_jit_charge_modmem(PAGE_SIZE);
	if (err)
		return ERR_PTR(err);
	image = arch_alloc_bpf_trampoline(PAGE_SIZE);
	if (!image) {
		bpf_jit_uncharge_modmem(PAGE_SIZE);
		return ERR_PTR(-ENOMEM);
	}

	return image;
}

void bpf_struct_ops_image_free(void *image)
{
	if (image) {
		arch_free_bpf_trampoline(image, PAGE_SIZE);
		bpf_jit_uncharge_modmem(PAGE_SIZE);
	}
}

#define MAYBE_NULL_SUFFIX "__nullable"