path: root/drivers/vhost/vdpa.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2018-2020 Intel Corporation.
 * Copyright (C) 2020 Red Hat, Inc.
 *
 * Author: Tiwei Bie <tiwei.bie@intel.com>
 *         Jason Wang <jasowang@redhat.com>
 *
 * Thanks Michael S. Tsirkin for the valuable comments and
 * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
 * their supports.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/mm.h>
#include <linux/iommu.h>
#include <linux/uuid.h>
#include <linux/vdpa.h>
#include <linux/nospec.h>
#include <linux/vhost.h>
#include <linux/virtio_net.h>

#include "vhost.h"

enum {
	VHOST_VDPA_BACKEND_FEATURES =
	(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
	(1ULL << VHOST_BACKEND_F_IOTLB_BATCH),
};

#define VHOST_VDPA_DEV_MAX (1U << MINORBITS)

struct vhost_vdpa {
	struct vhost_dev vdev;
	struct iommu_domain *domain;
	struct vhost_virtqueue *vqs;
	struct completion completion;
	struct vdpa_device *vdpa;
	struct device dev;
	struct cdev cdev;
	atomic_t opened;
	int nvqs;
	int virtio_id;
	int minor;
	struct eventfd_ctx *config_ctx;
	int in_batch;
	struct vdpa_iova_range range;
};

static DEFINE_IDA(vhost_vdpa_ida);

static dev_t vhost_vdpa_major;

static void handle_vq_kick(struct vhost_work *work)
{
	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
						  poll.work);
	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
	const struct vdpa_config_ops *ops = v->vdpa->config;

	ops->kick_vq(v->vdpa, vq - v->vqs);
}

static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
{
	struct vhost_virtqueue *vq = private;
	struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;

	if (call_ctx)
		eventfd_signal(call_ctx, 1);

	return IRQ_HANDLED;
}

static irqreturn_t vhost_vdpa_config_cb(void *private)
{
	struct vhost_vdpa *v = private;
	struct eventfd_ctx *config_ctx = v->config_ctx;

	if (config_ctx)
		eventfd_signal(config_ctx, 1);

	return IRQ_HANDLED;
}

static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
{
	struct vhost_virtqueue *vq = &v->vqs[qid];
	const struct vdpa_config_ops *ops = v->vdpa->config;
	struct vdpa_device *vdpa = v->vdpa;
	int ret, irq;

	if (!ops->get_vq_irq)
		return;

	irq = ops->get_vq_irq(vdpa, qid);
	if (irq < 0)
		return;

	irq_bypass_unregister_producer(&vq->call_ctx.producer);
	if (!vq->call_ctx.ctx)
		return;

	vq->call_ctx.producer.token = vq->call_ctx.ctx;
	vq->call_ctx.producer.irq = irq;
	ret = irq_bypass_register_producer(&vq->call_ctx.producer);
	if (unlikely(ret))
		dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret =  %d\n",
			 qid, vq->call_ctx.producer.token, ret);
}

static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
{
	struct vhost_virtqueue *vq = &v->vqs[qid];

	irq_bypass_unregister_producer(&vq->call_ctx.producer);
}

static void vhost_vdpa_reset(struct vhost_vdpa *v)
{
	struct vdpa_device *vdpa = v->vdpa;

	vdpa_reset(vdpa);
	v->in_batch = 0;
}

static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
{
	struct vdpa_device *vdpa = v->vdpa;
	const struct vdpa_config_ops *ops = vdpa->config;
	u32 device_id;

	device_id = ops->get_device_id(vdpa);

	if (copy_to_user(argp, &device_id, sizeof(device_id)))
		return -EFAULT;

	return 0;
}

static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
{
	struct vdpa_device *vdpa = v->vdpa;
	const struct vdpa_config_ops *ops = vdpa