summaryrefslogtreecommitdiff
path: root/io_uring
diff options
context:
space:
mode:
Diffstat (limited to 'io_uring')
-rw-r--r--io_uring/advise.c8
-rw-r--r--io_uring/cancel.c4
-rw-r--r--io_uring/epoll.c4
-rw-r--r--io_uring/fs.c28
-rw-r--r--io_uring/io-wq.c3
-rw-r--r--io_uring/io_uring.c19
-rw-r--r--io_uring/kbuf.c10
-rw-r--r--io_uring/msg_ring.c8
-rw-r--r--io_uring/net.c66
-rw-r--r--io_uring/notif.c4
-rw-r--r--io_uring/notif.h2
-rw-r--r--io_uring/openclose.c16
-rw-r--r--io_uring/poll.c16
-rw-r--r--io_uring/rsrc.c10
-rw-r--r--io_uring/rw.c28
-rw-r--r--io_uring/splice.c8
-rw-r--r--io_uring/sqpoll.c4
-rw-r--r--io_uring/statx.c6
-rw-r--r--io_uring/sync.c12
-rw-r--r--io_uring/timeout.c26
-rw-r--r--io_uring/uring_cmd.c17
-rw-r--r--io_uring/xattr.c18
22 files changed, 170 insertions, 147 deletions
diff --git a/io_uring/advise.c b/io_uring/advise.c
index 581956934c0b..449c6f14649f 100644
--- a/io_uring/advise.c
+++ b/io_uring/advise.c
@@ -31,7 +31,7 @@ struct io_madvise {
int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
- struct io_madvise *ma = io_kiocb_to_cmd(req);
+ struct io_madvise *ma = io_kiocb_to_cmd(req, struct io_madvise);
if (sqe->buf_index || sqe->off || sqe->splice_fd_in)
return -EINVAL;
@@ -48,7 +48,7 @@ int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
- struct io_madvise *ma = io_kiocb_to_cmd(req);
+ struct io_madvise *ma = io_kiocb_to_cmd(req, struct io_madvise);
int ret;
if (issue_flags & IO_URING_F_NONBLOCK)
@@ -64,7 +64,7 @@ int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_fadvise *fa = io_kiocb_to_cmd(req);
+ struct io_fadvise *fa = io_kiocb_to_cmd(req, struct io_fadvise);
if (sqe->buf_index || sqe->addr || sqe->splice_fd_in)
return -EINVAL;
@@ -77,7 +77,7 @@ int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_fadvise *fa = io_kiocb_to_cmd(req);
+ struct io_fadvise *fa = io_kiocb_to_cmd(req, struct io_fadvise);
int ret;
if (issue_flags & IO_URING_F_NONBLOCK) {
diff --git a/io_uring/cancel.c b/io_uring/cancel.c
index 8435a1eba59a..e4e1dc0325f0 100644
--- a/io_uring/cancel.c
+++ b/io_uring/cancel.c
@@ -107,7 +107,7 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,
int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_cancel *cancel = io_kiocb_to_cmd(req);
+ struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
if (unlikely(req->flags & REQ_F_BUFFER_SELECT))
return -EINVAL;
@@ -164,7 +164,7 @@ static int __io_async_cancel(struct io_cancel_data *cd,
int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_cancel *cancel = io_kiocb_to_cmd(req);
+ struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
struct io_cancel_data cd = {
.ctx = req->ctx,
.data = cancel->addr,
diff --git a/io_uring/epoll.c b/io_uring/epoll.c
index a8b794471d6b..9aa74d2c80bc 100644
--- a/io_uring/epoll.c
+++ b/io_uring/epoll.c
@@ -23,7 +23,7 @@ struct io_epoll {
int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_epoll *epoll = io_kiocb_to_cmd(req);
+ struct io_epoll *epoll = io_kiocb_to_cmd(req, struct io_epoll);
pr_warn_once("%s: epoll_ctl support in io_uring is deprecated and will "
"be removed in a future Linux kernel version.\n",
@@ -49,7 +49,7 @@ int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_epoll *ie = io_kiocb_to_cmd(req);
+ struct io_epoll *ie = io_kiocb_to_cmd(req, struct io_epoll);
int ret;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
diff --git a/io_uring/fs.c b/io_uring/fs.c
index 0de4f549bb7d..7100c293c13a 100644
--- a/io_uring/fs.c
+++ b/io_uring/fs.c
@@ -49,7 +49,7 @@ struct io_link {
int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_rename *ren = io_kiocb_to_cmd(req);
+ struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename);
const char __user *oldf, *newf;
if (sqe->buf_index || sqe->splice_fd_in)
@@ -79,7 +79,7 @@ int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_rename *ren = io_kiocb_to_cmd(req);
+ struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename);
int ret;
if (issue_flags & IO_URING_F_NONBLOCK)
@@ -95,7 +95,7 @@ int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
void io_renameat_cleanup(struct io_kiocb *req)
{
- struct io_rename *ren = io_kiocb_to_cmd(req);
+ struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename);
putname(ren->oldpath);
putname(ren->newpath);
@@ -103,7 +103,7 @@ void io_renameat_cleanup(struct io_kiocb *req)
int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_unlink *un = io_kiocb_to_cmd(req);
+ struct io_unlink *un = io_kiocb_to_cmd(req, struct io_unlink);
const char __user *fname;
if (sqe->off || sqe->len || sqe->buf_index || sqe->splice_fd_in)
@@ -128,7 +128,7 @@ int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_unlink *un = io_kiocb_to_cmd(req);
+ struct io_unlink *un = io_kiocb_to_cmd(req, struct io_unlink);
int ret;
if (issue_flags & IO_URING_F_NONBLOCK)
@@ -146,14 +146,14 @@ int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
void io_unlinkat_cleanup(struct io_kiocb *req)
{
- struct io_unlink *ul = io_kiocb_to_cmd(req);
+ struct io_unlink *ul = io_kiocb_to_cmd(req, struct io_unlink);
putname(ul->filename);
}
int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_mkdir *mkd = io_kiocb_to_cmd(req);
+ struct io_mkdir *mkd = io_kiocb_to_cmd(req, struct io_mkdir);
const char __user *fname;
if (sqe->off || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
@@ -175,7 +175,7 @@ int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_mkdir *mkd = io_kiocb_to_cmd(req);
+ struct io_mkdir *mkd = io_kiocb_to_cmd(req, struct io_mkdir);
int ret;
if (issue_flags & IO_URING_F_NONBLOCK)
@@ -190,14 +190,14 @@ int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags)
void io_mkdirat_cleanup(struct io_kiocb *req)
{
- struct io_mkdir *md = io_kiocb_to_cmd(req);
+ struct io_mkdir *md = io_kiocb_to_cmd(req, struct io_mkdir);
putname(md->filename);
}
int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_link *sl = io_kiocb_to_cmd(req);
+ struct io_link *sl = io_kiocb_to_cmd(req, struct io_link);
const char __user *oldpath, *newpath;
if (sqe->len || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
@@ -225,7 +225,7 @@ int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_link *sl = io_kiocb_to_cmd(req);
+ struct io_link *sl = io_kiocb_to_cmd(req, struct io_link);
int ret;
if (issue_flags & IO_URING_F_NONBLOCK)
@@ -240,7 +240,7 @@ int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags)
int io_linkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_link *lnk = io_kiocb_to_cmd(req);
+ struct io_link *lnk = io_kiocb_to_cmd(req, struct io_link);
const char __user *oldf, *newf;
if (sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
@@ -270,7 +270,7 @@ int io_linkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_linkat(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_link *lnk = io_kiocb_to_cmd(req);
+ struct io_link *lnk = io_kiocb_to_cmd(req, struct io_link);
int ret;
if (issue_flags & IO_URING_F_NONBLOCK)
@@ -286,7 +286,7 @@ int io_linkat(struct io_kiocb *req, unsigned int issue_flags)
void io_link_cleanup(struct io_kiocb *req)
{
- struct io_link *sl = io_kiocb_to_cmd(req);
+ struct io_link *sl = io_kiocb_to_cmd(req, struct io_link);
putname(sl->oldpath);
putname(sl->newpath);
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index 77df5b43bf52..c6536d4b2da0 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -624,8 +624,6 @@ static int io_wqe_worker(void *data)
snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task->pid);
set_task_comm(current, buf);
- audit_alloc_kernel(current);
-
while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
long ret;
@@ -660,7 +658,6 @@ static int io_wqe_worker(void *data)
if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
io_worker_handle_work(worker);
- audit_free(current);
io_worker_exit(worker);
return 0;
}
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index b54218da075c..ebfdb2212ec2 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -3885,13 +3885,15 @@ out_fput:
static int __init io_uring_init(void)
{
-#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
+#define __BUILD_BUG_VERIFY_OFFSET_SIZE(stype, eoffset, esize, ename) do { \
BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
- BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
+ BUILD_BUG_ON(sizeof_field(stype, ename) != esize); \
} while (0)
#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
- __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
+ __BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, sizeof(etype), ename)
+#define BUILD_BUG_SQE_ELEM_SIZE(eoffset, esize, ename) \
+ __BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, esize, ename)
BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
BUILD_BUG_SQE_ELEM(0, __u8, opcode);
BUILD_BUG_SQE_ELEM(1, __u8, flags);
@@ -3899,6 +3901,8 @@ static int __init io_uring_init(void)
BUILD_BUG_SQE_ELEM(4, __s32, fd);
BUILD_BUG_SQE_ELEM(8, __u64, off);
BUILD_BUG_SQE_ELEM(8, __u64, addr2);
+ BUILD_BUG_SQE_ELEM(8, __u32, cmd_op);
+ BUILD_BUG_SQE_ELEM(12, __u32, __pad1);
BUILD_BUG_SQE_ELEM(16, __u64, addr);
BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
BUILD_BUG_SQE_ELEM(24, __u32, len);
@@ -3917,13 +3921,22 @@ static int __init io_uring_init(void)
BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
+ BUILD_BUG_SQE_ELEM(28, __u32, rename_flags);
+ BUILD_BUG_SQE_ELEM(28, __u32, unlink_flags);
+ BUILD_BUG_SQE_ELEM(28, __u32, hardlink_flags);
+ BUILD_BUG_SQE_ELEM(28, __u32, xattr_flags);
+ BUILD_BUG_SQE_ELEM(28, __u32, msg_ring_flags);
BUILD_BUG_SQE_ELEM(32, __u64, user_data);
BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
BUILD_BUG_SQE_ELEM(40, __u16, buf_group);
BUILD_BUG_SQE_ELEM(42, __u16, personality);
BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
BUILD_BUG_SQE_ELEM(44, __u32, file_index);
+ BUILD_BUG_SQE_ELEM(44, __u16, notification_idx);
+ BUILD_BUG_SQE_ELEM(46, __u16, addr_len);
BUILD_BUG_SQE_ELEM(48, __u64, addr3);
+ BUILD_BUG_SQE_ELEM_SIZE(48, 0, cmd);
+ BUILD_BUG_SQE_ELEM(56, __u64, __pad2);
BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
sizeof(struct io_uring_rsrc_update));
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index e538fa7cb727..25cd724ade18 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -272,7 +272,7 @@ void io_destroy_buffers(struct io_ring_ctx *ctx)
int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_provide_buf *p = io_kiocb_to_cmd(req);
+ struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
u64 tmp;
if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||
@@ -291,7 +291,7 @@ int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_provide_buf *p = io_kiocb_to_cmd(req);
+ struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
struct io_ring_ctx *ctx = req->ctx;
struct io_buffer_list *bl;
int ret = 0;
@@ -319,7 +319,7 @@ int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
unsigned long size, tmp_check;
- struct io_provide_buf *p = io_kiocb_to_cmd(req);
+ struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
u64 tmp;
if (sqe->rw_flags || sqe->splice_fd_in)
@@ -421,7 +421,7 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_provide_buf *p = io_kiocb_to_cmd(req);
+ struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
struct io_ring_ctx *ctx = req->ctx;
struct io_buffer_list *bl;
int ret = 0;
@@ -436,7 +436,7 @@ int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
bl = io_buffer_get_list(ctx, p->bgid);
if (unlikely(!bl)) {
- bl = kzalloc(sizeof(*bl), GFP_KERNEL);
+ bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);
if (!bl) {
ret = -ENOMEM;
goto err;
diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
index 753d16734319..976c4ba68ee7 100644
--- a/io_uring/msg_ring.c
+++ b/io_uring/msg_ring.c
@@ -26,7 +26,7 @@ struct io_msg {
static int io_msg_ring_data(struct io_kiocb *req)
{
struct io_ring_ctx *target_ctx = req->file->private_data;
- struct io_msg *msg = io_kiocb_to_cmd(req);
+ struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
if (msg->src_fd || msg->dst_fd || msg->flags)
return -EINVAL;
@@ -76,7 +76,7 @@ static int io_double_lock_ctx(struct io_ring_ctx *ctx,
static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_ring_ctx *target_ctx = req->file->private_data;
- struct io_msg *msg = io_kiocb_to_cmd(req);
+ struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
struct io_ring_ctx *ctx = req->ctx;
unsigned long file_ptr;
struct file *src_file;
@@ -122,7 +122,7 @@ out_unlock:
int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_msg *msg = io_kiocb_to_cmd(req);
+ struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
if (unlikely(sqe->buf_index || sqe->personality))
return -EINVAL;
@@ -141,7 +141,7 @@ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_msg *msg = io_kiocb_to_cmd(req);
+ struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
int ret;
ret = -EBADFD;
diff --git a/io_uring/net.c b/io_uring/net.c
index 32fc3da04e41..6d71748e2c5a 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -70,13 +70,14 @@ struct io_sendzc {
unsigned flags;
unsigned addr_len;
void __user *addr;
+ size_t done_io;
};
#define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED)
int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
+ struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
sqe->buf_index || sqe->splice_fd_in))
@@ -88,7 +89,7 @@ int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_shutdown *shutdown = io_kiocb_to_cmd(req);
+ struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
struct socket *sock;
int ret;
@@ -173,7 +174,7 @@ static int io_setup_async_msg(struct io_kiocb *req,
static int io_sendmsg_copy_hdr(struct io_kiocb *req,
struct io_async_msghdr *iomsg)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
iomsg->msg.msg_name = &iomsg->addr;
iomsg->free_iov = iomsg->fast_iov;
@@ -200,7 +201,7 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
if (unlikely(sqe->file_index || sqe->addr2))
return -EINVAL;
@@ -224,7 +225,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr iomsg, *kmsg;
struct socket *sock;
unsigned flags;
@@ -283,7 +284,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
int io_send(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct msghdr msg;
struct iovec iov;
struct socket *sock;
@@ -357,7 +358,7 @@ static bool io_recvmsg_multishot_overflow(struct io_async_msghdr *iomsg)
static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
struct io_async_msghdr *iomsg)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct user_msghdr msg;
int ret;
@@ -404,7 +405,7 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
struct io_async_msghdr *iomsg)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct compat_msghdr msg;
struct compat_iovec __user *uiov;
int ret;
@@ -482,7 +483,7 @@ int io_recvmsg_prep_async(struct io_kiocb *req)
int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
if (unlikely(sqe->file_index || sqe->addr2))
return -EINVAL;
@@ -517,7 +518,7 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
static inline void io_recv_prep_retry(struct io_kiocb *req)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
sr->done_io = 0;
sr->len = 0; /* get from the provided buffer */
@@ -575,12 +576,12 @@ static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg,
if (kmsg->controllen) {
unsigned long control = ubuf + hdr - kmsg->controllen;
- kmsg->msg.msg_control_user = (void *) control;
+ kmsg->msg.msg_control_user = (void __user *) control;
kmsg->msg.msg_controllen = kmsg->controllen;
}
sr->buf = *buf; /* stash for later copy */
- *buf = (void *) (ubuf + hdr);
+ *buf = (void __user *) (ubuf + hdr);
kmsg->payloadlen = *len = *len - hdr;
return 0;
}
@@ -646,7 +647,7 @@ static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io,
int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct io_async_msghdr iomsg, *kmsg;
struct socket *sock;
unsigned int cflags;
@@ -758,7 +759,7 @@ retry_multishot:
int io_recv(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req);
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct msghdr msg;
struct socket *sock;
struct iovec iov;
@@ -849,7 +850,7 @@ out_free:
int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_sendzc *zc = io_kiocb_to_cmd(req);
+ struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);
struct io_ring_ctx *ctx = req->ctx;
if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3))
@@ -878,6 +879,7 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
zc->addr_len = READ_ONCE(sqe->addr_len);
+ zc->done_io = 0;
#ifdef CONFIG_COMPAT
if (req->ctx->compat)
@@ -944,7 +946,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
{
struct sockaddr_storage address;
struct io_ring_ctx *ctx = req->ctx;
- struct io_sendzc *zc = io_kiocb_to_cmd(req);
+ struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);
struct io_notif_slot *notif_slot;
struct io_kiocb *notif;
struct msghdr msg;
@@ -1012,18 +1014,30 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
if (unlikely(ret < min_ret)) {
if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
return -EAGAIN;
- return ret == -ERESTARTSYS ? -EINTR : ret;
+ if (ret > 0 && io_net_retry(sock, msg.msg_flags)) {
+ zc->len -= ret;
+ zc->buf += ret;
+ zc->done_io += ret;
+ req->flags |= REQ_F_PARTIAL_IO;
+ return -EAGAIN;
+ }
+ if (ret == -ERESTARTSYS)
+ ret = -EINTR;
+ } else if (zc->flags & IORING_RECVSEND_NOTIF_FLUSH) {
+ io_notif_slot_flush_submit(notif_slot, 0);
}
- if (zc->flags & IORING_RECVSEND_NOTIF_FLUSH)
- io_notif_slot_flush_submit(notif_slot, 0);
+ if (ret >= 0)
+ ret += zc->done_io;
+ else if (zc->done_io)
+ ret = zc->done_io;
io_req_set_res(req, ret, 0);
return IOU_OK;
}
int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_accept *accept = io_kiocb_to_cmd(req);
+ struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
unsigned flags;
if (sqe->len || sqe->buf_index)
@@ -1057,7 +1071,7 @@ int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_accept(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_ring_ctx *ctx = req->ctx;
- struct io_accept *accept = io_kiocb_to_cmd(req);
+ struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
bool fixed = !!accept->file_slot;
@@ -1115,7 +1129,7 @@ retry:
int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_socket *sock = io_kiocb_to_cmd(req);
+ struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
if (sqe->addr || sqe->rw_flags || sqe->buf_index)
return -EINVAL;
@@ -1136,7 +1150,7 @@ int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_socket(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_socket *sock = io_kiocb_to_cmd(req);
+ struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
bool fixed = !!sock->file_slot;
struct file *file;
int ret, fd;
@@ -1170,14 +1184,14 @@ int io_socket(struct io_kiocb *req, unsigned int issue_flags)
int io_connect_prep_async(struct io_kiocb *req)
{
struct io_async_connect *io = req->async_data;
- struct io_connect *conn = io_kiocb_to_cmd(req);
+ struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
}
int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_connect *conn = io_kiocb_to_cmd(req);
+ struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
return -EINVAL;
@@ -1189,7 +1203,7 @@ int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_connect(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_connect *connect = io_kiocb_to_cmd(req);
+ struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect);
struct io_async_connect __io, *io;
unsigned file_flags;
int ret;
diff --git a/io_uring/notif.c b/io_uring/notif.c
index b5f989dff9de..977736e82c1a 100644
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -100,7 +100,7 @@ __cold int io_notif_unregister(struct io_ring_ctx *ctx)
if (!notif)
continue;
- nd = io_kiocb_to_cmd(notif);
+ nd = io_notif_to_data(notif);
slot->notif = NULL;
if (!refcount_dec_and_test(&nd->uarg.refcnt))
continue;
@@ -123,8 +123,6 @@ __cold int io_notif_register(struct io_ring_ctx *ctx,
struct io_uring_notification_register reg;
unsigned i;
- BUILD_BUG_ON(sizeof(struct io_notif_data) > 64);
-
if (ctx->nr_notif_slots)
return -EBUSY;
if (size != sizeof(reg))
diff --git a/io_uring/notif.h b/io_uring/notif.h
index 0819304d7e00..65f0b42f2555 100644
--- a/io_uring/notif.h
+++ b/io_uring/notif.h
@@ -46,7 +46,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx,
static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif)
{
- return io_kiocb_to_cmd(notif);
+ return io_kiocb_to_cmd(notif, struct io_notif_data);
}
static inline struct io_kiocb *io_get_notif(struct io_ring_ctx *ctx,
diff --git a/io_uring/openclose.c b/io_uring/openclose.c
index d1818ec9169b..67178e4bb282 100644
--- a/io_uring/openclose.c
+++ b/io_uring/openclose.c
@@ -33,7 +33,7 @@ struct io_close {
static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_open *open = io_kiocb_to_cmd(req);
+ struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
const char __user *fname;
int ret;
@@ -66,7 +66,7 @@ static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe
int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_open *open = io_kiocb_to_cmd(req);
+ struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
u64 mode = READ_ONCE(sqe->len);
u64 flags = READ_ONCE(sqe->open_flags);
@@ -76,7 +76,7 @@ int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_open *open = io_kiocb_to_cmd(req);
+ struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
struct open_how __user *how;
size_t len;
int ret;
@@ -95,7 +95,7 @@ int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_open *open = io_kiocb_to_cmd(req);
+ struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
struct open_flags op;
struct file *file;
bool resolve_nonblock, nonblock_set;
@@ -167,7 +167,7 @@ int io_openat(struct io_kiocb *req, unsigned int issue_flags)
void io_open_cleanup(struct io_kiocb *req)
{
- struct io_open *open = io_kiocb_to_cmd(req);
+ struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
if (open->filename)
putname(open->filename);
@@ -187,14 +187,14 @@ int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags,
static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_close *close = io_kiocb_to_cmd(req);
+ struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1);
}
int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_close *close = io_kiocb_to_cmd(req);
+ struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index)
return -EINVAL;
@@ -212,7 +212,7 @@ int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_close(struct io_kiocb *req, unsigned int issue_flags)
{
struct files_struct *files = current->files;
- struct io_close *close = io_kiocb_to_cmd(req);
+ struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
struct fdtable *fdt;
struct file *file;
int ret = -EBADF;
diff --git a/io_uring/poll.c b/io_uring/poll.c
index dadd293749b0..d5bad0bea6e4 100644
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -85,7 +85,7 @@ static struct io_poll *io_poll_get_double(struct io_kiocb *req)
static struct io_poll *io_poll_get_single(struct io_kiocb *req)
{
if (req->opcode == IORING_OP_POLL_ADD)
- return io_kiocb_to_cmd(req);
+ return io_kiocb_to_cmd(req, struct io_poll);
return &req->apoll->poll;
}
@@ -274,7 +274,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)
return;
if (ret == IOU_POLL_DONE) {
- struct io_poll *poll = io_kiocb_to_cmd(req);
+ struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);
req->cqe.res = mangle_poll(req->cqe.res & poll->events);
} else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
req->cqe.res = ret;
@@ -475,7 +475,7 @@ static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
struct poll_table_struct *p)
{
struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
- struct io_poll *poll = io_kiocb_to_cmd(pt->req);
+ struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll);
__io_queue_proc(poll, pt, head,
(struct io_poll **) &pt->req->async_data);
@@ -821,7 +821,7 @@ static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_poll_update *upd = io_kiocb_to_cmd(req);
+ struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update);
u32 flags;
if (sqe->buf_index || sqe->splice_fd_in)
@@ -851,7 +851,7 @@ int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- struct io_poll *poll = io_kiocb_to_cmd(req);
+ struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);
u32 flags;
if (sqe->buf_index || sqe->off || sqe->addr)
@@ -868,7 +868,7 @@ int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_poll *poll = io_kiocb_to_cmd(req);
+ struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);
struct io_poll_table ipt;
int ret;
@@ -891,7 +891,7 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
{
- struct io_poll_update *poll_update = io_kiocb_to_cmd(req);