path: root/arch/loongarch/kvm/mmu.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
 */

#include <linux/highmem.h>
#include <linux/hugetlb.h>
#include <linux/kvm_host.h>
#include <linux/page-flags.h>
#include <linux/uaccess.h>
#include <asm/mmu_context.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/kvm_mmu.h>

static inline bool kvm_hugepage_capable(struct kvm_memory_slot *slot)
{
	return slot->arch.flags & KVM_MEM_HUGEPAGE_CAPABLE;
}

static inline bool kvm_hugepage_incapable(struct kvm_memory_slot *slot)
{
	return slot->arch.flags & KVM_MEM_HUGEPAGE_INCAPABLE;
}

static inline void kvm_ptw_prepare(struct kvm *kvm, kvm_ptw_ctx *ctx)
{
	ctx->level = kvm->arch.root_level;
	/* pte table */
	ctx->invalid_ptes  = kvm->arch.invalid_ptes;
	ctx->pte_shifts    = kvm->arch.pte_shifts;
	ctx->pgtable_shift = ctx->pte_shifts[ctx->level];
	ctx->invalid_entry = ctx->invalid_ptes[ctx->level];
	ctx->opaque        = kvm;
}

/*
 * Mark a range of guest physical address space old (all accesses fault) in the
 * VM's GPA page table to allow detection of commonly used pages.
 */
static int kvm_mkold_pte(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx)
{
	if (kvm_pte_young(*pte)) {
		*pte = kvm_pte_mkold(*pte);
		return 1;
	}

	return 0;
}

/*
 * Mark a range of guest physical address space clean (writes fault) in the VM's
 * GPA page table to allow dirty page tracking.
 */
static int kvm_mkclean_pte(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx)
{
	gfn_t offset;
	kvm_pte_t val;

	val = *pte;
	/*
	 * For kvm_arch_mmu_enable_log_dirty_pt_masked with mask, start and end
	 * may cross hugepage, for first huge page parameter addr is equal to
	 * start, however for the second huge page addr is base address of
	 * this huge page, rather than start or end address
	 */
	if ((ctx->flag & _KVM_HAS_PGMASK) && !kvm_pte_huge(val)) {
		offset = (addr >> PAGE_SHIFT) - ctx->gfn;
		if (!(BIT(offset) & ctx->mask))
			return 0;
	}

	/*
	 * Need not split huge page now, just set write-proect pte bit
	 * Split huge page until next write fault
	 */
	if (kvm_pte_dirty(val)) {
		*pte = kvm_pte_mkclean(val);
		return 1;
	}

	return 0;
}

/*
 * Clear pte entry
 */
static int kvm_flush_pte(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx)
{
	struct kvm *kvm;

	kvm = ctx->opaque;
	if (ctx->level)
		kvm->stat.hugepages--;
	else
		kvm->stat.pages--;

	*pte = ctx->invalid_entry;

	return 1;
}

/*
 * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory.
 *
 * Allocate a blank KVM GPA page directory (PGD) for representing guest physical
 * to host physical page mappings.
 *
 * Returns:	Pointer to new KVM GPA page directory.
 *		NULL on allocation failure.
 */
kvm_pte_t *kvm_pgd_alloc(void)
{
	kvm_pte_t *pgd;

	pgd = (kvm_pte_t *)__get_free_pages(GFP_KERNEL, 0);
	if (pgd)
		pgd_init((void *)pgd);

	return pgd;
}

static void _kvm_pte_init(void *addr, unsigned long val)
{
	unsigned long *p, *end;

	p = (unsigned long *)addr;
	end = p + PTRS_PER_PTE;
	do {
		p[0] = val;
		p[1] = val;
		p[2] = val;
		p[3] = val;
		p[4] = val;
		p += 8;
		p[-3] = val;
		p[-2] = val;
		p[-1] = val;
	} while (p != end);
}

/*
 * Caller must hold kvm->mm_lock
 *
 * Walk the page tables of kvm to find the PTE corresponding to the
 * address @addr. If page tables don't exist for @addr, they will be created
 * from the MMU cache if @cache is not NULL.
 */
static kvm_pte_t *kvm_populate_gpa(struct kvm *kvm,
				struct kvm_mmu_memory_cache *cache,
				unsigned long addr, int level)
{
	kvm_ptw_ctx ctx;
	kvm_pte_t *entry, *child;

	kvm_ptw_prepare(kvm, &ctx);
	child = kvm->arch.pgd;
	while (ctx.level > level) {
		entry = kvm_pgtable_offset(&ctx, child, addr);
		if (kvm_pte_none(&ctx, entry)) {
			if (!cache)
				return NULL;

			child = kvm_mmu_memory_cache_alloc(cache);
			_kvm_pte_init(child, ctx.invalid_ptes[ctx.level - 1]);
			kvm_set_pte(entry, __pa(child));
		} else if (kvm_pte_huge(*entry)) {
			return entry;
		} else
			child = (kvm_pte_t *)__va(PHYSADDR(*entry));
		kvm_ptw_enter(&ctx);
	}

	entry = kvm_pgtable_offset(&ctx, child, addr);

	return entry;
}

/*
 * Page walker for VM shadow mmu at last level
 * The last level is small pte page or huge pmd page
 */
static int kvm_ptw_leaf(kvm_pte_t *dir, phys_addr_t addr, phys_addr_t end, kvm_ptw_ctx *ctx)
{
	int ret;
	phys_addr_t next, start, size;
	struct list_head *list;
	kvm_pte_t *entry, *child;

	ret = 0;
	start = addr;
	child = (kvm_pte_t *)__va(PHYSADDR(*dir));
	entry = kvm_pgtable_offset(ctx, child, addr);
	do {
		next = addr + (0x1UL << ctx->pgtable_shift);
		if (!kvm_pte_present(ctx, entry))
			continue;

		ret |= ctx->ops(entry, addr, ctx);
	} while (entry++, addr = next, addr < end);

	if (kvm_need_flush(ctx)) {
		size