// SPDX-License-Identifier: GPL-2.0-only
/*
* Helpers for the host side of a virtio ring.
*
* Since these may be in userspace, we use (inline) accessors.
*/
#include <linux/compiler.h>
#include <linux/module.h>
#include <linux/vringh.h>
#include <linux/virtio_ring.h>
#include <linux/kernel.h>
#include <linux/ratelimit.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
#include <linux/export.h>
#if IS_REACHABLE(CONFIG_VHOST_IOTLB)
#include <linux/bvec.h>
#include <linux/highmem.h>
#include <linux/vhost_iotlb.h>
#endif
#include <uapi/linux/virtio_config.h>
static __printf(1,2) __cold void vringh_bad(const char *fmt, ...)
{
static DEFINE_RATELIMIT_STATE(vringh_rs,
DEFAULT_RATELIMIT_INTERVAL,
DEFAULT_RATELIMIT_BURST);
if (__ratelimit(&vringh_rs)) {
va_list ap;
va_start(ap, fmt);
printk(KERN_NOTICE "vringh:");
vprintk(fmt, ap);
va_end(ap);
}
}
/* Returns vring->num if empty, -ve on error. */
static inline int __vringh_get_head(const struct vringh *vrh,
int (*getu16)(const struct vringh *vrh,
u16 *val, const __virtio16 *p),
u16 *last_avail_idx)
{
u16 avail_idx, i, head;
int err;
err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);
if (err) {
vringh_bad("Failed to access avail idx at %p",
&vrh->vring.avail->idx);
return err;
}
if (*last_avail_idx == avail_idx)
return vrh->vring.num;
/* Only get avail ring entries after they have been exposed by guest. */
virtio_rmb(vrh->weak_barriers);
i = *last_avail_idx & (vrh