path: root/io_uring/net.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/net.h>
#include <linux/compat.h>
#include <net/compat.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "kbuf.h"
#include "alloc_cache.h"
#include "net.h"
#include "notif.h"
#include "rsrc.h"

#if defined(CONFIG_NET)
struct io_shutdown {
	struct file			*file;
	int				how;
};

struct io_accept {
	struct file			*file;
	struct sockaddr __user		*addr;
	int __user			*addr_len;
	int				flags;
	u32				file_slot;
	unsigned long			nofile;
};

struct io_socket {
	struct file			*file;
	int				domain;
	int				type;
	int				protocol;
	int				flags;
	u32				file_slot;
	unsigned long			nofile;
};

struct io_connect {
	struct file			*file;
	struct sockaddr __user		*addr;
	int				addr_len;
	bool				in_progress;
	bool				seen_econnaborted;
};

struct io_sr_msg {
	struct file			*file;
	union {
		struct compat_msghdr __user	*umsg_compat;
		struct user_msghdr __user	*umsg;
		void __user			*buf;
	};
	unsigned			len;
	unsigned			done_io;
	unsigned			msg_flags;
	unsigned			nr_multishot_loops;
	u16				flags;
	/* initialised and used only by !msg send variants */
	u16				addr_len;
	u16				buf_group;
	void __user			*addr;
	void __user			*msg_control;
	/* used only for send zerocopy */
	struct io_kiocb 		*notif;
};

/*
 * Number of times we'll try and do receives if there's more data. If we
 * exceed this limit, then add us to the back of the queue and retry from
 * there. This helps fairness between flooding clients.
 */
#define MULTISHOT_MAX_RETRY	32

static inline bool io_check_multishot(struct io_kiocb *req,
				      unsigned int issue_flags)
{
	/*
	 * When ->locked_cq is set we only allow to post CQEs from the original
	 * task context. Usual request completions will be handled in other
	 * generic paths but multipoll may decide to post extra cqes.
	 */
	return !(issue_flags & IO_URING_F_IOWQ) ||
		!(issue_flags & IO_URING_F_MULTISHOT) ||
		!req->ctx->task_complete;
}

int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);

	if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
		     sqe->buf_index || sqe->splice_fd_in))
		return -EINVAL;

	shutdown->how = READ_ONCE(sqe->len);
	req->flags |= REQ_F_FORCE_ASYNC;
	return 0;
}

int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
	struct socket *sock;
	int ret;

	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	ret = __sys_shutdown_sock(sock, shutdown->how);
	io_req_set_res(req, ret, 0);
	return IOU_OK;
}

static bool io_net_retry(struct socket *sock, int flags)
{
	if (!(flags & MSG_WAITALL))
		return false;
	return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
}

static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_async_msghdr *hdr = req->async_data;

	if (!req_has_async_data(req) || issue_flags & IO_URING_F_UNLOCKED)
		return;

	/* Let normal cleanup path reap it if we fail adding to the cache */
	if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) {
		req->async_data = NULL;
		req->flags &= ~REQ_F_ASYNC_DATA;
	}
}

static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req,
						  unsigned int issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_cache_entry *entry;
	struct io_async_msghdr *hdr;

	if (!(issue_flags & IO_URING_F_UNLOCKED)) {
		entry = io_alloc_cache_get(&ctx->netmsg_cache);
		if (entry) {
			hdr = container_of(entry, struct io_async_msghdr, cache);
			hdr->free_iov = NULL;
			req->flags |= REQ_F_ASYNC_DATA;
			req->async_data = hdr;
			return hdr;
		}
	}

	if (!io_alloc_async_data(req)) {
		hdr = req->async_data;
		hdr->free_iov = NULL;
		return hdr;
	}
	return NULL;
}

static inline struct io_async_msghdr *io_msg_alloc_async_prep(struct io_kiocb *req)
{
	/* ->prep_async is always called from the submission context */
	return io_msg_alloc_async(req, 0);
}

static int io_setup_async_msg(struct io_kiocb *req,
			      struct io_async_msghdr *kmsg,
			      unsigned int issue_flags)
{
	struct io_async_msghdr *async_msg;

	if (req_has_async_data(req))
		return -EAGAIN;
	async_msg = io_msg_alloc_async(req, issue_flags);
	if (!async_msg) {
		kfree(kmsg->free_iov);
		return -ENOMEM;
	}
	req->flags |= REQ_F_NEED_CLEANUP;
	memcpy(async_msg, kmsg, sizeof(*kmsg));
	if (async_msg->msg.msg_name)
		async_msg->msg.msg_name = &async_msg->addr;

	if ((req->flags & REQ_F_BUFFER_SELECT) && !async_msg->msg.msg_iter.nr_segs)
		return -EAGAIN;

	/* if were using fast_iov, set it to the new one */
	if (iter_is_iovec(&kmsg->msg.msg_iter) && !kmsg->free_iov) {
		size_t fast_idx = iter_iov(&kmsg->msg.msg_iter) - kmsg->fast_iov;
		async_msg->msg.msg_iter.__iov = &async_msg->fast_iov[fast_idx];
	}

	return -EAGAIN;
}

static int io_sendmsg_copy_hdr(struct io_kiocb *req,
			       struct io_async_msghdr *iomsg)
{
	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
	int ret;

	iomsg->msg.msg_name = &iomsg->addr;
	iomsg->free_iov = iomsg->fast_iov;
	ret = sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags,
					&iomsg->free_iov);
	/* save msg_control as sys_sendmsg() overwrites it */
	sr->msg_control = iomsg->msg.msg_control_user;
	return ret;
}

int io_send_prep_async(struct io_kiocb *req)
{
	struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
	struct io_async_msghdr *io;
	int ret;

	if (!zc->addr || req_has_async_data(req))
		return 0;
	io = io_msg_alloc_async_prep(req);
	if (!io)
		return -ENOMEM;
	ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr);
	return ret;
}

static int io_setup_async_addr(struct io_kiocb *req,
			      struct sockaddr_storage *addr_storage,
			      unsigned int issue_flags)
{
	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
	struct io_async_msghdr *io;

	if (!sr->addr || req_has_async_data(req))
		return -EAGAIN;
	io = io_msg_alloc_async(req, issue_flags);
	if (!io)
		return -ENOMEM;
	memcpy(&io->addr, addr_storage, sizeof(io->addr));
	return -EAGAIN;
}

int io_sendmsg_prep_async(struct io_kiocb *req)
{
	int ret;

	if (!io_msg_alloc_async_prep(req))
		return -ENOMEM;
	ret = io_sendmsg_copy_hdr(req, req->async_data);
	if (!ret)
		req->flags |= REQ_F_NEED_CLEANUP;
	return ret;
}

void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
{
	struct io_async_msghdr *io = req->async_data;

	kfree(io->free_iov);
}

int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

	if (req->opcode == IORING_OP_SEND) {
		if (READ_ONCE(sqe->__pad3[0]))
			return -EINVAL;
		sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
		sr->addr_len = READ_ONCE(sqe->addr_len);
	} else if (sqe->addr2 || sqe->file_index) {
		return -EINVAL;
	}

	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
	sr->len = READ_ONCE(sqe->len);
	sr->flags = READ_ONCE(s