// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2018-2020 Intel Corporation.
* Copyright (C) 2020 Red Hat, Inc.
*
* Author: Tiwei Bie <tiwei.bie@intel.com>
* Jason Wang <jasowang@redhat.com>
*
* Thanks Michael S. Tsirkin for the valuable comments and
* suggestions. And thanks to Cunming Liang and Zhihong Wang for all
* their supports.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/iommu.h>
#include <linux/uuid.h>
#include <linux/vdpa.h>
#include <linux/nospec.h>
#include <linux/vhost.h>
#include "vhost.h"
enum {
VHOST_VDPA_BACKEND_FEATURES =
(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
(1ULL << VHOST_BACKEND_F_IOTLB_BATCH) |
(1ULL << VHOST_BACKEND_F_IOTLB_ASID),
};
#define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
#define VHOST_VDPA_IOTLB_BUCKETS 16
struct vhost_vdpa_as {
struct hlist_node hash_link;
struct vhost_iotlb iotlb;
u32 id;
};
struct vhost_vdpa {
struct vhost_dev vdev;
struct iommu_domain *domain;
struct vhost_virtqueue *vqs;
struct completion completion;
struct vdpa_device *vdpa;
struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS];
struct device dev;
struct cdev cdev;
atomic_t opened;
u32 nvqs;
int virtio_id;
int minor;
struct eventfd_ctx *config_ctx;
int in_batch;
struct vdpa_iova_range range;
u32 batch_asid;
bool suspended;
};
static DEFINE_IDA(vhost_vdpa_ida);
static dev_t vhost_vdpa_major;
static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
struct vhost_iotlb *iotlb, u64 start,
u64 last, u32 asid);
static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
{
struct vhost_vdpa_as *as = container_of(iotlb, struct
vhost_vdpa_as, iotlb);
return as->id;
}
static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)
{
struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
struct vhost_vdpa_as *as;
hlist_for_each_entry(as, head, hash_link)
if (as->id == asid)
return as;
return NULL;
}
static struct vhost_iotlb *asid_to_iotlb(struct vho
| 