path: root/net/core/sock_map.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */

#include <linux/bpf.h>
#include <linux/btf_ids.h>
#include <linux/filter.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/net.h>
#include <linux/workqueue.h>
#include <linux/skmsg.h>
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/sock_diag.h>
#include <net/udp.h>

struct bpf_stab {
	struct bpf_map map;
	struct sock **sks;
	struct sk_psock_progs progs;
	raw_spinlock_t lock;
};

#define SOCK_CREATE_FLAG_MASK				\
	(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)

static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
{
	struct bpf_stab *stab;

	if (!capable(CAP_NET_ADMIN))
		return ERR_PTR(-EPERM);
	if (attr->max_entries == 0 ||
	    attr->key_size    != 4 ||
	    (attr->value_size != sizeof(u32) &&
	     attr->value_size != sizeof(u64)) ||
	    attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
		return ERR_PTR(-EINVAL);

	stab = kzalloc(sizeof(*stab), GFP_USER | __GFP_ACCOUNT);
	if (!stab)
		return ERR_PTR(-ENOMEM);

	bpf_map_init_from_attr(&stab->map, attr);
	raw_spin_lock_init(&stab->lock);

	stab->sks = bpf_map_area_alloc(stab->map.max_entries *
				       sizeof(struct sock *),
				       stab->map.numa_node);
	if (!stab->sks) {
		kfree(stab);
		return ERR_PTR(-ENOMEM);
	}

	return &stab->map;
}

int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog)
{
	u32 ufd = attr->target_fd;
	struct bpf_map *map;
	struct fd f;
	int ret;

	if (attr->attach_flags || attr->replace_bpf_fd)
		return -EINVAL;

	f = fdget(ufd);
	map = __bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);
	ret = sock_map_prog_update(map, prog, NULL, attr->attach_type);
	fdput(f);
	return ret;
}

int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
{
	u32 ufd = attr->target_fd;
	struct bpf_prog *prog;
	struct bpf_map *map;
	struct fd f;
	int ret;

	if (attr->attach_flags || attr->replace_bpf_fd)
		return -EINVAL;

	f = fdget(ufd);
	map = __bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	prog = bpf_prog_get(attr->attach_bpf_fd);
	if (IS_ERR(prog)) {
		ret = PTR_ERR(prog);
		goto put_map;
	}

	if (prog->type != ptype) {
		ret = -EINVAL;
		goto put_prog;
	}

	ret = sock_map_prog_update(map, NULL, prog, attr->attach_type);
put_prog:
	bpf_prog_put(prog);
put_map:
	fdput(f);
	return ret;
}

static void sock_map_sk_acquire(struct sock *sk)
	__acquires(&sk->sk_lock.slock)
{
	lock_sock(sk);
	preempt_disable();
	rcu_read_lock();
}

static void sock_map_sk_release(struct sock *sk)
	__releases(&sk->sk_lock.slock)
{
	rcu_read_unlock();
	preempt_enable();
	release_sock(sk);
}

static void sock_map_add_link(struct sk_psock *psock,
			      struct sk_psock_link *link,
			      struct bpf_map *map, void *link_raw)
{
	link->link_raw = link_raw;
	link->map = map;
	spin_lock_bh(&psock->link_lock);
	list_add_tail(&link->list, &psock->link);
	spin_unlock_bh(&psock->link_lock);
}

static void sock_map_del_link(struct sock *sk,
			      struct sk_psock *psock, void *link_raw)
{
	bool strp_stop = false, verdict_stop = false;
	struct sk_psock_link *link, *tmp;

	spin_lock_bh(&psock->link_lock);
	list_for_each_entry_safe(link, tmp, &psock->link, list) {
		if (link->link_raw == link_raw) {
			struct bpf_map *map = link->map;
			struct bpf_stab *stab = container_of(map, struct bpf_stab,
							     map);
			if (psock->parser.enabled && stab->progs.skb_parser)
				strp_stop = true;
			if (psock->parser.enabled && stab->progs.skb_verdict)
				verdict_stop = true;
			list_del(&link->list);
			sk_psock_free_link(link);
		}
	}
	spin_unlock_bh(&psock->link_lock);
	if (strp_stop || verdict_stop) {
		write_lock_bh(&sk->sk_callback_lock);
		if (strp_stop)
			sk_psock_stop_strp(sk, psock);
		else
			sk_psock_stop_verdict(sk, psock);
		write_unlock_bh(&sk->sk_callback_lock);
	}
}

static void sock_map_unref(struct sock *sk, void *link_raw)
{
	struct sk_psock *psock = sk_psock(sk);

	if (likely(psock)) {
		sock_map_del_link(sk, psock, link_raw);
		sk_psock_put(sk, psock);
	}
}

static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)
{
	struct proto *prot;

	switch (sk->sk_type) {
	case SOCK_STREAM:
		prot = tcp_bpf_get_proto(sk, psock);
		break;

	case SOCK_DGRAM:
		prot = udp_bpf_get_proto(sk, psock);
		break;

	default:
		return -EINVAL;
	}

	if (IS_ERR(prot))
		return PTR_ERR(prot);

	sk_psock_update_proto(sk, psock, prot);
	return 0;
}

static struct sk_psock *sock_map_psock_get_checked(struct sock *sk)
{
	struct sk_psock *psock;

	rcu_read_lock();
	psock = sk_psock(sk);
	if (psock) {
		if (sk->sk_prot->close != sock_map_close) {
			psock = ERR_PTR(-EBU