path: root/io_uring/net.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/net.h>
#include <linux/compat.h>
#include <net/compat.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "kbuf.h"
#include "alloc_cache.h"
#include "net.h"
#include "notif.h"
#include "rsrc.h"

#if defined(CONFIG_NET)
struct io_shutdown {
	struct file			*file;
	int				how;
};

struct io_accept {
	struct file			*file;
	struct sockaddr __user		*addr;
	int __user			*addr_len;
	int				flags;
	u32				file_slot;
	unsigned long			nofile;
};

struct io_socket {
	struct file			*file;
	int				domain;
	int				type;
	int				protocol;
	int				flags;
	u32				file_slot;
	unsigned long			nofile;
};

struct io_connect {
	struct file			*file;
	struct sockaddr __user		*addr;
	int				addr_len;
	bool				in_progress;
	bool				seen_econnaborted;
};

struct io_sr_msg {
	struct file			*file;
	union {
		struct compat_msghdr __user	*umsg_compat;
		struct user_msghdr __user	*umsg;
		void __user			*buf;
	};
	unsigned			len;
	unsigned			done_io;
	unsigned			msg_flags;
	unsigned			nr_multishot_loops;
	u16				flags;
	/* initialised and used only by !msg send variants */
	u16				addr_len;
	u16				buf_group;
	void __user			*addr;
	void __user			*msg_control;
	/* used only for send zerocopy */
	struct io_kiocb 		*notif;
};

/*
 * Number of times we'll try and do receives if there's more data. If we
 * exceed this limit, then add us to the back of the queue and retry from
 * there. This helps fairness between flooding clients.
 */
#define MULTISHOT_MAX_RETRY	32

int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);

	if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
		     sqe->buf_index || sqe->splice_fd_in))
		return -EINVAL;

	shutdown->how = READ_ONCE(sqe->len);
	req->flags |= REQ_F_FORCE_ASYNC;
	return 0;
}

int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
	struct socket *sock;
	int ret;

	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	ret = __sys_shutdown_sock(sock, shutdown->how);
	io_req_set_res(req, ret, 0);
	return IOU_OK;
}

static bool io_net_retry(struct socket *sock, int flags)
{
	if (!(flags & MSG_WAITALL))
		return false;
	return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
}

static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)
{
	if (kmsg->free_iov) {
		kfree(kmsg->free_iov);
		kmsg->free_iov_nr = 0;
		kmsg->free_iov = NULL;
	}
}

static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_async_msghdr *hdr = req->async_data;
	struct iovec *iov;

	/* can't recycle, ensure we free the iovec if we have one */
	if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
		io_netmsg_iovec_free(hdr);
		return;
	}

	/* Let normal cleanup path reap it if we fail adding to the cache */
	iov = hdr->free_iov;
	if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) {
		if (iov)
			kasan_mempool_poison_object(iov);
		req->async_data = NULL;
		req->flags &= ~REQ_F_ASYNC_DATA;
	}
}

static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_async_msghdr *hdr;

	hdr = io_alloc_cache_get(&ctx->netmsg_cache);
	if (hdr) {
		if (hdr->free_iov) {
			kasan_mempool_unpoison_object(hdr->free_iov,
				hdr->free_iov_nr * sizeof(struct iovec));
			req->flags |= REQ_F_NEED_CLEANUP;
		}
		req->flags |= REQ_F_ASYNC_DATA;
		req->async_data = hdr;
		return hdr;
	}

	if (!io_alloc_async_data(req)) {
		hdr = req->async_data;
		hdr->free_iov_nr = 0;
		hdr->free_iov = NULL;
		return hdr;
	}
	return NULL;
}

/* assign new iovec to kmsg, if we need to */
static int io_net_vec_assign(struct io_kiocb *req, struct io_async_msghdr *kmsg,
			     struct iovec *iov)
{
	if (iov) {
		req->flags |= REQ_F_NEED_CLEANUP;
		kmsg->free_iov_nr = kmsg->msg.msg_iter.nr_segs;
		if (kmsg->free_iov)
			kfree(kmsg->free_iov);
		kmsg->free_iov = iov;
	}
	return 0;
}

static inline void io_mshot_prep_retry(struct io_kiocb *req,
				       struct io_async_msghdr *kmsg)
{
	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

	req->flags &= ~REQ_F_BL_EMPTY;
	sr->done_io = 0;
	sr->len = 0; /* get from the provided buffer */
	req->buf_index = sr->buf_group;
}

#ifdef CONFIG_COMPAT
static int io_compat_msg_copy_hdr(struct io_kiocb *req,
				  struct io_async_msghdr *iomsg,
				  struct compat_msghdr *msg, int ddir)
{
	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
	struct compat_iovec __user *uiov;
	struct iovec *iov;
	int ret, nr_segs;

	if (iomsg->free_iov) {
		nr_segs = iomsg->free_iov_nr;
		iov = iomsg->free_iov;
	} else {
		iov = &iomsg->fast_iov;
		nr_segs = 1;
	}

	if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))
		return -EFAULT;

	uiov = compat_ptr(msg->msg_iov);
	if (req->flags & REQ_F_BUFFER_SELECT) {
		compat_ssize_t clen;

		if (msg->msg_iovlen == 0) {
			sr->len = iov->iov_len = 0;
			iov->iov_base = NULL;
		} else if (msg->msg_iovlen > 1) {
			return -EINVAL;
		} else {
			if (!access_ok(uiov, sizeof(*uiov)))
				return -EFAULT;
			if (__get_user(clen, &uiov->iov_len))
				return -EFAULT;
			if (clen < 0)
				return -EINVAL;
			sr->len = clen;
		}

		return 0;
	}

	ret = __import_iovec(ddir, (struct iovec __user *)uiov, msg->msg_iovlen,
				nr_segs, &iov, &iomsg->msg.msg_iter, true);
	if (unlikely(ret < 0))
		return ret;

	return io_net_vec_assign(req, iomsg, iov);
}
#endif

static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
			   struct user_msghdr *msg, int ddir)
{
	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
	struct iovec *iov;
	int ret, nr_segs;

	if (iomsg->free_iov) {
		nr_segs = iomsg->free_iov_nr;
		iov = iomsg->free_iov;
	} else {
		iov = &iomsg->fast_iov;
		nr_segs = 1;
	}

	if (!user_access_begin(sr->umsg, sizeof(*sr->umsg)))
		return -EFAULT;

	ret = -EFAULT;
	unsafe_get_user(msg->msg_name, &sr->umsg->msg_name, ua_end);
	unsafe_get_user(msg->msg_namelen, &sr->umsg->msg_namelen, ua_end);
	unsafe_get_user(msg->msg_iov, &sr->umsg->msg_iov, ua_end);
	unsafe_get_user(msg->msg_iovlen, &sr->umsg->msg_iovlen, ua_end);
	unsafe_get_user(msg->msg_control, &sr->umsg->msg_control, ua_end);
	unsafe_get_user(msg->msg_controllen, &sr->umsg->msg_controllen, ua_end);
	msg->msg_flags = 0;

	if (req->flags & REQ_F_BUFFER_SELECT) {
		if (msg->msg_iovlen == 0) {
			sr->len = iov->iov_len = 0;
			iov->iov_base = NULL;
		} else if (msg->msg_iovlen > 1) {
			ret = -EINVAL;
			goto ua_end;
		} else {
			/* we only need the length for provided buffers */
			if (!access_ok(&msg->msg_iov[0].iov_len, sizeof(__kernel_size_t)))
				goto ua_end;
			unsafe_get_user(iov->iov_len, &msg->msg_iov[0].iov_len,
					ua_end);
			sr->len = iov->iov_len;
		}
		ret = 0;
ua_end:
		user_access_end();
		return ret;
	}

	user_access_end();
	ret = __import_iovec(ddir, msg->msg_iov, msg->msg_iovlen, nr_segs,
				&iov, &iomsg->msg.msg_iter, false);
	if (unlikely(ret < 0))
		return ret;

	return io_net_vec_assign(req, iomsg, iov);
}

static int io_sendmsg_copy_hdr(struct io_kiocb *req,
			       struct io_async_msghdr *iomsg)
{
	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
	struct user_msghdr msg;
	int ret;

	iomsg->msg.msg_name = &iomsg->addr;
	iomsg->msg.msg_iter.nr_segs = 0;

#ifdef CONFIG_COMPAT
	if (unlikely(req->ctx->compat)) {
		struct compat_msghdr cmsg;

		ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ITER_SOURCE);
		if (unlikely(ret))
			return ret;

		return __get_compat_msghdr(&iomsg-&g