path: root/mm/hmm.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright 2013 Red Hat Inc.
 *
 * Authors: Jérôme Glisse <jglisse@redhat.com>
 */
/*
 * Refer to include/linux/hmm.h for information about heterogeneous memory
 * management or HMM for short.
 */
#include <linux/mm.h>
#include <linux/hmm.h>
#include <linux/init.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/mmzone.h>
#include <linux/pagemap.h>
#include <linux/swapops.h>
#include <linux/hugetlb.h>
#include <linux/memremap.h>
#include <linux/jump_label.h>
#include <linux/dma-mapping.h>
#include <linux/mmu_notifier.h>
#include <linux/memory_hotplug.h>

#define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)

#if IS_ENABLED(CONFIG_HMM_MIRROR)
static const struct mmu_notifier_ops hmm_mmu_notifier_ops;

static inline struct hmm *mm_get_hmm(struct mm_struct *mm)
{
	struct hmm *hmm = READ_ONCE(mm->hmm);

	if (hmm && kref_get_unless_zero(&hmm->kref))
		return hmm;

	return NULL;
}

/**
 * hmm_get_or_create - register HMM against an mm (HMM internal)
 *
 * @mm: mm struct to attach to
 * Returns: returns an HMM object, either by referencing the existing
 *          (per-process) object, or by creating a new one.
 *
 * This is not intended to be used directly by device drivers. If mm already
 * has an HMM struct then it get a reference on it and returns it. Otherwise
 * it allocates an HMM struct, initializes it, associate it with the mm and
 * returns it.
 */
static struct hmm *hmm_get_or_create(struct mm_struct *mm)
{
	struct hmm *hmm = mm_get_hmm(mm);
	bool cleanup = false;

	if (hmm)
		return hmm;

	hmm = kmalloc(sizeof(*hmm), GFP_KERNEL);
	if (!hmm)
		return NULL;
	init_waitqueue_head(&hmm->wq);
	INIT_LIST_HEAD(&hmm->mirrors);
	init_rwsem(&hmm->mirrors_sem);
	hmm->mmu_notifier.ops = NULL;
	INIT_LIST_HEAD(&hmm->ranges);
	mutex_init(&hmm->lock);
	kref_init(&hmm->kref);
	hmm->notifiers = 0;
	hmm->dead = false;
	hmm->mm = mm;

	spin_lock(&mm->page_table_lock);
	if (!mm->hmm)
		mm->hmm = hmm;
	else
		cleanup = true;
	spin_unlock(&mm->page_table_lock);

	if (cleanup)
		goto error;

	/*
	 * We should only get here if hold the mmap_sem in write mode ie on
	 * registration of first mirror through hmm_mirror_register()
	 */
	hmm->mmu_notifier.ops = &hmm_mmu_notifier_ops;
	if (__mmu_notifier_register(&hmm->mmu_notifier, mm))
		goto error_mm;

	return hmm;

error_mm:
	spin_lock(&mm->page_table_lock);
	if (mm->hmm == hmm)
		mm->hmm = NULL;
	spin_unlock(&mm->page_table_lock);
error:
	kfree(hmm);
	return NULL;
}

static void hmm_free(struct kref *kref)
{
	struct hmm *hmm = container_of(kref, struct hmm, kref);
	struct mm_struct *mm = hmm->mm;

	mmu_notifier_unregister_no_release(&hmm->mmu_notifier, mm);

	spin_lock(&mm->page_table_lock);
	if (mm->hmm == hmm)
		mm->hmm = NULL;
	spin_unlock(&mm->page_table_lock);

	kfree(hmm);
}

static inline void hmm_put(struct hmm *hmm)
{
	kref_put(&hmm->kref, hmm_free);
}

void hmm_mm_destroy(struct mm_struct *mm)
{
	struct hmm *hmm;

	spin_lock(&mm->page_table_lock);
	hmm = mm_get_hmm(mm);
	mm->hmm = NULL;
	if (hmm) {
		hmm->mm = NULL;
		hmm->dead = true;
		spin_unlock(&mm->page_table_lock);
		hmm_put(hmm);
		return;
	}

	spin_unlock(&mm->page_table_lock);
}

static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm)
{
	struct hmm *hmm = mm_get_hmm(mm);
	struct hmm_mirror *mirror;
	struct hmm_range *range;

	/* Report this HMM as dying. */
	hmm->dead = true;

	/* Wake-up everyone waiting on any range. */
	mutex_lock(&hmm->lock);
	list_for_each_entry(range, &hmm->ranges, list) {
		range->valid = false;
	}
	wake_up_all(&hmm->wq);
	mutex_unlock(&hmm->lock);

	down_write(&hmm->mirrors_sem);
	mirror = list_first_entry_or_null(&hmm->mirrors, struct hmm_mirror,
					  list);
	while (mirror) {
		list_del_init(&mirror->list);
		if (mirror->ops->release) {
			/*
			 * Drop mirrors_sem so callback can wait on any pending
			 * work that might itself trigger mmu_notifier callback
			 * and thus would deadlock with us.
			 */
			up_write(&hmm->mirrors_sem);
			mirror->ops->release(mirror);
			down_write(&hmm->mirrors_sem);
		}
		mirror = list_first_entry_or_null(&hmm->mirrors,
						  struct hmm_mirr