path: root/mm/hmm.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright 2013 Red Hat Inc.
 *
 * Authors: Jérôme Glisse <jglisse@redhat.com>
 */
/*
 * Refer to include/linux/hmm.h for information about heterogeneous memory
 * management or HMM for short.
 */
#include <linux/mm.h>
#include <linux/hmm.h>
#include <linux/init.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/mmzone.h>
#include <linux/pagemap.h>
#include <linux/swapops.h>
#include <linux/hugetlb.h>
#include <linux/memremap.h>
#include <linux/sched/mm.h>
#include <linux/jump_label.h>
#include <linux/dma-mapping.h>
#include <linux/mmu_notifier.h>
#include <linux/memory_hotplug.h>

static struct mmu_notifier *hmm_alloc_notifier(struct mm_struct *mm)
{
	struct hmm *hmm;

	hmm = kzalloc(sizeof(*hmm), GFP_KERNEL);
	if (!hmm)
		return ERR_PTR(-ENOMEM);

	init_waitqueue_head(&hmm->wq);
	INIT_LIST_HEAD(&hmm->mirrors);
	init_rwsem(&hmm->mirrors_sem);
	INIT_LIST_HEAD(&hmm->ranges);
	spin_lock_init(&hmm->ranges_lock);
	hmm->notifiers = 0;
	return &hmm->mmu_notifier;
}

static void hmm_free_notifier(struct mmu_notifier *mn)
{
	struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier);

	WARN_ON(!list_empty(&hmm->ranges));
	WARN_ON(!list_empty(&hmm->mirrors));
	kfree(hmm);
}

static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm)
{
	struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier);
	struct hmm_mirror *mirror;

	/*
	 * Since hmm_range_register() holds the mmget() lock hmm_release() is
	 * prevented as long as a range exists.
	 */
	WARN_ON(!list_empty_careful(&hmm->ranges));

	down_read(&hmm->mirrors_sem);
	list_for_each_entry(mirror, &hmm->mirrors, list) {
		/*
		 * Note: The driver is not allowed to trigger
		 * hmm_mirror_unregister() from this thread.
		 */
		if (mirror->ops->release)
			mirror->ops->release(mirror);
	}
	up_read(&hmm->mirrors_sem);
}

static void notifiers_decrement(struct hmm *hmm)
{
	unsigned long flags;

	spin_lock_irqsave(&hmm->ranges_lock, flags);
	hmm->notifiers--;
	if (!hmm->notifiers) {
		struct hmm_range *range;

		list_for_each_entry(range, &hmm->ranges, list) {
			if (range->valid)
				continue;
			range->valid = true;
		}
		wake_up_all(&hmm->wq);
	}
	spin_unlock_irqrestore(&hmm->ranges_lock, flags);
}

static int hmm_invalidate_range_start(struct mmu_notifier *mn,
			const struct mmu_notifier_range *nrange)
{
	struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier);
	struct hmm_mirror *mirror;
	struct hmm_range *range;
	unsigned long flags;
	int ret = 0;

	spin_lock_irqsave(&hmm->ranges_lock, flags);
	hmm->notifiers++;
	list_for_each_entry(range, &hmm->ranges, list) {
		if (nrange->end < range->start || nrange->start >= range->end)
			continue;

		range->valid = false;
	}
	spin_unlock_irqrestore(&hmm->ranges_lock, flags);

	if (mmu_notifier_range_blockable(nrange))
		down_read(&hmm->mirrors_sem);
	else if (!down_read_trylock(&hmm->mirrors_sem)) {
		ret = -EAGAIN;
		goto out;
	}

	list_for_each_entry(mirror, &hmm->mirrors, list) {
		int rc;

		rc = mirror->ops->sync_cpu_device_pagetables(mirror, nrange);
		if (rc) {
			if (WARN_ON(mmu_notifier_range_blockable(nrange) ||
			    rc != -EAGAIN))
				continue;
			ret = -EAGAIN;
			break;
		}
	}
	up_read(&hmm->mirrors_sem);

out:
	if (ret)
		notifiers_decrement(hmm);
	return ret;
}

static void hmm_invalidate_range_end(struct mmu_notifier *mn,
			const struct mmu_notifier_range *nrange)
{
	struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier);

	notifiers_decrement(hmm);
}

static const struct mmu_notifier_ops hmm_mmu_notifier_ops = {
	.release		= hmm_release,
	.invalidate_range_start	= hmm_invalidate_range_start,
	.invalidate_range_end	<