path: root/mm/pagewalk.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/pagewalk.h>
#include <linux/highmem.h>
#include <linux/sched.h>
#include <linux/hugetlb.h>
#include <linux/mmu_context.h>
#include <linux/swap.h>
#include <linux/swapops.h>

#include <asm/tlbflush.h>

#include "internal.h"

/*
 * We want to know the real level where a entry is located ignoring any
 * folding of levels which may be happening. For example if p4d is folded then
 * a missing entry found at level 1 (p4d) is actually at level 0 (pgd).
 */
static int real_depth(int depth)
{
	if (depth == 3 && PTRS_PER_PMD == 1)
		depth = 2;
	if (depth == 2 && PTRS_PER_PUD == 1)
		depth = 1;
	if (depth == 1 && PTRS_PER_P4D == 1)
		depth = 0;
	return depth;
}

static int walk_pte_range_inner(pte_t *pte, unsigned long addr,
				unsigned long end, struct mm_walk *walk)
{
	const struct mm_walk_ops *ops = walk->ops;
	int err = 0;

	for (;;) {
		if (ops->install_pte && pte_none(ptep_get(pte))) {
			pte_t new_pte;

			err = ops->install_pte(addr, addr + PAGE_SIZE, &new_pte,
					       walk);
			if (err)
				break;

			set_pte_at(walk->mm, addr, pte, new_pte);
			/* Non-present before, so for arches that need it. */
			if (!WARN_ON_ONCE(walk->no_vma))
				update_mmu_cache(walk->vma, addr, pte);
		} else {
			err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
			if (err)
				break;
		}
		if (addr >= end - PAGE_SIZE)
			break;
		addr += PAGE_SIZE;
		pte++;
	}
	return err;
}

static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
			  struct mm_walk *walk)
{
	pte_t *pte;
	int err = 0;
	spinlock_t *ptl;

	if (walk->no_vma) {
		/*
		 * pte_offset_map() might apply user-specific validation.
		 * Indeed, on x86_64 the pmd entries set up by init_espfix_ap()
		 * fit its pmd_bad() check (_PAGE_NX set and _PAGE_RW clear),
		 * and CONFIG_EFI_PGT_DUMP efi_mm goes so far as to walk them.
		 */
		if (walk->mm == &init_mm || addr >= TASK_SIZE)
			pte = pte_offset_kernel(pmd, addr);
		else
			pte = pte_offset_map(pmd, addr);
		if (pte) {
			err = walk_pte_range_inner(pte, addr, end, walk);
			if (walk->mm != &init_mm && addr < TASK_SIZE)
				pte_unmap(pte);
		}
	} else {
		pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
		if (pte) {
			err = walk_pte_range_inner(pte, addr, end, walk);
			pte_unmap_unlock(pte, ptl);
		}
	}
	if (!pte)
		walk->action = ACTION_AGAIN;
	return err;
}

static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
			  struct mm_walk *walk)
{
	pmd_t *pmd;
	unsigned long next;
	const struct mm_walk_ops *ops = walk->ops;
	bool has_handler = ops->pte_entry;
	bool has_install = ops->install_pte;
	int err = 0;
	int depth = real_depth(3);

	pmd = pmd_offset(pud, addr);
	do {
again:
		next = pmd_addr_end(addr, end);
		if (pmd_none(*pmd)) {
			if (has_install)
				err = __pte_alloc(walk->mm, pmd);
			else if (ops->pte_hole)
				err = ops->pte_hole(addr, next, depth, walk);
			if (err)
				break;
			if (!has_install)
				continue;
		}

		walk->action = ACTION_SUBTREE;

		/*
		 * This implies that each ->pmd_entry() handler
		 * needs to know about pmd_trans_huge() pmds
		 */
		if (ops->pmd_entry)
			err = ops->pmd_entry(pmd, addr, next, walk);
		if (err)
			break;

		if (walk->action == ACTION_AGAIN)
			goto again;
		if (walk->action == ACTION_CONTINUE)
			continue;

		if (!has_handler) { /* No handlers for lower page tables. */
			if (!has_install)
				continue; /* Nothing to do. */
			/*
			 * We are ONLY installing, so avoid unnecessarily
			 * splitting a present huge page.
			 */
			if (pmd_present(*pmd) &&
			    (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)))
				continue;
		}

		if (walk->vma)
			split_huge_pmd(walk->vma, pmd, addr);
		else if (pmd_leaf(*pmd) || !pmd_present(*pmd))
			continue; /* Nothing to do. */

		err = walk_pte_range(pmd, addr, next, walk);
		if (err)
			break;

		if (walk->action == ACTION_AGAIN)
			goto again;

	} while (pmd++, addr = next, addr != end);

	return err;
}

static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
			  struct mm_walk *walk)
{
	pud_t *pud;
	unsigned long next;
	const struct mm_walk_ops *ops = walk->ops;
	bool has_handler = ops->pmd_entry || ops->pte_entry;
	bool has_install = ops->install_pte;
	int err = 0;
	int depth = real_depth(2);

	pud = pud_offset(p4d, addr);
	do {
 again:
		next = pud_addr_end(addr, end);
		if (pud_none(*pud))</