// SPDX-License-Identifier: GPL-2.0
/*
* This is a module to test the HMM (Heterogeneous Memory Management)
* mirror and zone device private memory migration APIs of the kernel.
* Userspace programs can register with the driver to mirror their own address
* space and can use the device to read/write any valid virtual address.
*/
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/memremap.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/delay.h>
#include <linux/pagemap.h>
#include <linux/hmm.h>
#include <linux/vmalloc.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/sched/mm.h>
#include <linux/platform_device.h>
#include <linux/rmap.h>
#include <linux/mmu_notifier.h>
#include <linux/migrate.h>
#include "test_hmm_uapi.h"
#define DMIRROR_NDEVICES 4
#define DMIRROR_RANGE_FAULT_TIMEOUT 1000
#define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)
#define DEVMEM_CHUNKS_RESERVE 16
/*
* For device_private pages, dpage is just a dummy struct page
* representing a piece of device memory. dmirror_devmem_alloc_page
* allocates a real system memory page as backing storage to fake a
* real device. zone_device_data points to that backing page. But
* for device_coherent memory, the struct page represents real
* physical CPU-accessible memory that we can use directly.
*/
#define BACKING_PAGE(page) (is_device_private_page((page)) ? \
(page)->zone_device_data : (page))
static unsigned long spm_addr_dev0;
module_param(spm_addr_dev0, long, 0644);
MODULE_PARM_DESC(spm_addr_dev0,
"Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
static unsigned long spm_addr_dev1;
module_param(spm_addr_dev1, long, 0644);
MODULE_PARM_DESC(spm_addr_dev1,
"Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent
