path: root/arch/powerpc/lib/feature-fixups.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org)
 *
 *  Modifications for ppc64:
 *      Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com>
 *
 *  Copyright 2008 Michael Ellerman, IBM Corporation.
 */

#include <linux/types.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/stop_machine.h>
#include <asm/cputable.h>
#include <asm/code-patching.h>
#include <asm/interrupt.h>
#include <asm/page.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/security_features.h>
#include <asm/firmware.h>
#include <asm/inst.h>

/*
 * Used to generate warnings if mmu or cpu feature check functions that
 * use static keys before they are initialized.
 */
bool static_key_feature_checks_initialized __read_mostly;
EXPORT_SYMBOL_GPL(static_key_feature_checks_initialized);

struct fixup_entry {
	unsigned long	mask;
	unsigned long	value;
	long		start_off;
	long		end_off;
	long		alt_start_off;
	long		alt_end_off;
};

static u32 *calc_addr(struct fixup_entry *fcur, long offset)
{
	/*
	 * We store the offset to the code as a negative offset from
	 * the start of the alt_entry, to support the VDSO. This
	 * routine converts that back into an actual address.
	 */
	return (u32 *)((unsigned long)fcur + offset);
}

static int patch_alt_instruction(u32 *src, u32 *dest, u32 *alt_start, u32 *alt_end)
{
	int err;
	ppc_inst_t instr;

	instr = ppc_inst_read(src);

	if (instr_is_relative_branch(ppc_inst_read(src))) {
		u32 *target = (u32 *)branch_target(src);

		/* Branch within the section doesn't need translating */
		if (target < alt_start || target > alt_end) {
			err = translate_branch(&instr, dest, src);
			if (err)
				return 1;
		}
	}

	raw_patch_instruction(dest, instr);

	return 0;
}

static int patch_feature_section_mask(unsigned long value, unsigned long mask,
				      struct fixup_entry *fcur)
{
	u32 *start, *end, *alt_start, *alt_end, *src, *dest;

	start = calc_addr(fcur, fcur->start_off);
	end = calc_addr(fcur, fcur->end_off);
	alt_start = calc_addr(fcur, fcur->alt_start_off);
	alt_end = calc_addr(fcur, fcur->alt_end_off);

	if ((alt_end - alt_start) > (end - start))
		return 1;

	if ((value & fcur->mask & mask) == (fcur->value & mask))
		return 0;

	src = alt_start;
	dest = start;

	for (; src < alt_end; src = ppc_inst_next(src, src),
			      dest = ppc_inst_next(dest, dest)) {
		if (patch_alt_instruction(src, dest, alt_start, alt_end))
			return 1;
	}

	for (; dest < end; dest++)
		raw_patch_instruction(dest, ppc_inst(PPC_RAW_NOP()));

	return 0;
}

static void do_feature_fixups_mask(unsigned long value, unsigned long mask,
				   void *fixup_start, void *fixup_end)
{
	struct fixup_entry *fcur, *fend;

	fcur = fixup_start;
	fend = fixup_end;

	for (; fcur < fend; fcur++) {
		if (patch_feature_section_mask(value, mask, fcur)) {
			WARN_ON(1);
			printk("Unable to patch feature section at %p - %p" \
				" with %p - %p\n",
				calc_addr(fcur, fcur->start_off),
				calc_addr(fcur, fcur->end_off),
				calc_addr(fcur, fcur->alt_start_off),
				calc_addr(fcur, fcur->alt_end_off));
		}
	}
}

void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
{
	do_feature_fixups_mask(value, ~0, fixup_start, fixup_end);
}

#ifdef CONFIG_PPC_BARRIER_NOSPEC
static bool is_fixup_addr_valid(void *dest, size_t size)
{
	return system_state < SYSTEM_FREEING_INITMEM ||
	       !init_section_contains(dest, size);
}

static int do_patch_fixups(long *start, long *end, unsigned int *instrs, int num)
{
	int i;

	for (i = 0; start < end; start++, i++) {
		int j;
		unsigned int *dest = (void *)start + *start;

		if (!is_fixup_addr_valid(dest, sizeof(*instrs) * num))
			continue;

		pr_devel("patching dest %lx\n", (unsigned long)dest);

		for (j = 0; j < num; j++)
			patch_instruction(dest + j, ppc_inst(instrs[j]));
	}
	return i;
}
#endif

#ifdef CONFIG_PPC_BOOK3S_64
static int do_patch_entry_fixups(long *start, long *end, unsigned int *instrs,
				 bool do_fallback, void *fallback)
{
	int i;

	for (i = 0; start < end; start++, i++) {
		unsigned int *dest = (void *)start + *start;

		if (!is_fixup_addr_valid(dest, sizeof(*instrs) * 3))
			continue;

		pr_devel("patching dest %lx\n", (unsigned long)dest);

		// See comment in do_entry_flush_fixups() RE order of patching
		if (do_fallbac