path: root/arch/x86/kernel/cpu/mtrr/generic.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
 * because MTRRs can span up to 40 bits (36bits on most modern x86)
 */

#include <linux/export.h>
#include <linux/init.h>
#include <linux/io.h>
#include <linux/mm.h>
#include <linux/cc_platform.h>
#include <asm/processor-flags.h>
#include <asm/cacheinfo.h>
#include <asm/cpufeature.h>
#include <asm/hypervisor.h>
#include <asm/mshyperv.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
#include <asm/msr.h>
#include <asm/memtype.h>

#include "mtrr.h"

struct fixed_range_block {
	int base_msr;		/* start address of an MTRR block */
	int ranges;		/* number of MTRRs in this block  */
};

static struct fixed_range_block fixed_range_blocks[] = {
	{ MSR_MTRRfix64K_00000, 1 }, /* one   64k MTRR  */
	{ MSR_MTRRfix16K_80000, 2 }, /* two   16k MTRRs */
	{ MSR_MTRRfix4K_C0000,  8 }, /* eight  4k MTRRs */
	{}
};

struct cache_map {
	u64 start;
	u64 end;
	u64 flags;
	u64 type:8;
	u64 fixed:1;
};

bool mtrr_debug;

static int __init mtrr_param_setup(char *str)
{
	int rc = 0;

	if (!str)
		return -EINVAL;
	if (!strcmp(str, "debug"))
		mtrr_debug = true;
	else
		rc = -EINVAL;

	return rc;
}
early_param("mtrr", mtrr_param_setup);

/*
 * CACHE_MAP_MAX is the maximum number of memory ranges in cache_map, where
 * no 2 adjacent ranges have the same cache mode (those would be merged).
 * The number is based on the worst case:
 * - no two adjacent fixed MTRRs share the same cache mode
 * - one variable MTRR is spanning a huge area with mode WB
 * - 255 variable MTRRs with mode UC all overlap with the WB MTRR, creating 2
 *   additional ranges each (result like "ababababa...aba" with a = WB, b = UC),
 *   accounting for MTRR_MAX_VAR_RANGES * 2 - 1 range entries
 * - a TOP_MEM2 area (even with overlapping an UC MTRR can't add 2 range entries
 *   to the possible maximum, as it always starts at 4GB, thus it can't be in
 *   the middle of that MTRR, unless that MTRR starts at 0, which would remove
 *   the initial "a" from the "abababa" pattern above)
 * The map won't contain ranges with no matching MTRR (those fall back to the
 * default cache mode).
 */
#define CACHE_MAP_MAX	(MTRR_NUM_FIXED_RANGES + MTRR_MAX_VAR_RANGES * 2)

static struct cache_map init_cache_map[CACHE_MAP_MAX] __initdata;
static struct cache_map *cache_map __refdata = init_cache_map;
static unsigned int cache_map_size = CACHE_MAP_MAX;
static unsigned int cache_map_n;
static unsigned int cache_map_fixed;

static unsigned long smp_changes_mask;
static int mtrr_state_set;
u64 mtrr_tom2;

struct mtrr_state_type mtrr_state;
EXPORT_SYMBOL_GPL(mtrr_state);

/* Reserved bits in the high portion of the MTRRphysBaseN MSR. */
u32 phys_hi_rsvd;

/*
 * BIOS is expected to clear MtrrFixDramModEn bit, see for example
 * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
 * Opteron Processors" (26094 Rev. 3.30 February 2006), section
 * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set
 * to 1 during BIOS initialization of the fixed MTRRs, then cleared to
 * 0 for operation."
 */
static inline void k8_check_syscfg_dram_mod_en(void)
{
	u32 lo, hi;

	if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
	      (boot_cpu_data.x86 >= 0x0f)))
		return;

	if (cc_platform_has(CC_ATTR_HOST_SEV_SNP))
		return;

	rdmsr(MSR_AMD64_SYSCFG, lo, hi);
	if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
		pr_err(FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
		       " not cleared by BIOS, clearing this bit\n",
		       smp_processor_id());
		lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
		mtrr_wrmsr(MSR_AMD64_SYSCFG, lo, hi);
	}
}

/* Get the size of contiguous MTRR range */
static u64 get_mtrr_size(u64 mask)
{
	u64 size;

	mask |= (u64)phys_hi_rsvd << 32;
	size = -mask;

	return size;
}

static u8 get_var_mtrr_state(unsigned int reg, u64 *start, u64 *size)
{
	struct mtrr_var_range *mtrr = mtrr_state.var_ranges + reg;

	if (!(mtrr->mask_lo & MTRR_PHYSMASK_V))
		return MTRR_TYPE_INVALID;

	*start = (((u64)mtrr->base_hi) << 32) + (mtrr->base_lo & PAGE_MASK);
	*size = get_mtrr_size((((u64)mtrr->mask_hi) << 32) +
			      (mtrr->mask_lo & PAGE_MASK));

	return mtrr->base_lo & MTRR_PHYSBASE_TYPE;
}

static u8 get_effective_type(u8 type1, u8 type2)
{
	if (type1 == MTRR_TYPE_UNCACHABLE || type2 == MTRR_TYPE_UNCACHABLE)
		return MTRR_TYPE_UNCACHABLE;

	if ((type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH) ||
	    (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK))
		return MTRR_TYPE_WRTHROUGH;

	if (type1 != type2)
		return MTRR_TYPE_UNCACHABLE;

	return type1;
}

static void rm_map_entry_at(int idx)
{
	cache_map_n--;
	if (cache_map_n > idx) {
		memmove(cache_map + idx, cache_map + idx + 1,
			sizeof(*cache_map) * (cache_map_n - idx));
	}
}

/*
 * Add an entry into cache_map at a specific index.  Merges adjacent entries if
 * appropriate.  Return the number of merges for correcting the scan index
 * (this is needed as merging will reduce the number of entries, which will
 * result in skipping entries in future iterations if the scan index isn't
 * corrected).
 * Note that the corrected index can never go below -1 (resulting in being 0 in
 * the next scan iteration), as "2" is returned only if the current index is
 * larger than zero.
 */
static int add_map_entry_at(u64 start, u64 end, u8 type, int idx)
{
	bool merge_prev = false, merge_next = false;

	if (start >= end)
		return 0;

	if (idx > 0) {
		struct cache_map *prev = cache_map + idx - 1