path: root/arch/powerpc/lib/feature-fixups.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org)
 *
 *  Modifications for ppc64:
 *      Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com>
 *
 *  Copyright 2008 Michael Ellerman, IBM Corporation.
 */

#include <linux/types.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/stop_machine.h>
#include <asm/cputable.h>
#include <asm/code-patching.h>
#include <asm/interrupt.h>
#include <asm/page.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/security_features.h>
#include <asm/firmware.h>
#include <asm/inst.h>

struct fixup_entry {
	unsigned long	mask;
	unsigned long	value;
	long		start_off;
	long		end_off;
	long		alt_start_off;
	long		alt_end_off;
};

static u32 *calc_addr(struct fixup_entry *fcur, long offset)
{
	/*
	 * We store the offset to the code as a negative offset from
	 * the start of the alt_entry, to support the VDSO. This
	 * routine converts that back into an actual address.
	 */
	return (u32 *)((unsigned long)fcur + offset);
}

static int patch_alt_instruction(u32 *src, u32 *dest, u32 *alt_start, u32 *alt_end)
{
	int err;
	ppc_inst_t instr;

	instr = ppc_inst_read(src);

	if (instr_is_relative_branch(ppc_inst_read(src))) {
		u32 *target = (u32 *)branch_target(src);

		/* Branch within the section doesn't need translating */
		if (target < alt_start || target > alt_end) {
			err = translate_branch(&instr, dest, src);
			if (err)
				return 1;
		}
	}

	raw_patch_instruction(dest, instr);

	return 0;
}

static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
{
	u32 *start, *end, *alt_start, *alt_end, *src, *dest;

	start = calc_addr(fcur, fcur->start_off);
	end = calc_addr(fcur, fcur->end_off);
	alt_start = calc_addr(fcur, fcur->alt_start_off);
	alt_end = calc_addr(fcur, fcur->alt_end_off);

	if ((alt_end - alt_start) > (end - start))
		return 1;

	if ((value & fcur->mask) == fcur->value)
		return 0;

	src = alt_start;
	dest = start;

	for (; src < alt_end; src = ppc_inst_next(src, src),
			      dest = ppc_inst_next(dest, dest)) {
		if (patch_alt_instruction(src, dest, alt_start, alt_end))
			return 1;
	}

	for (; dest < end; dest++)
		raw_patch_instruction(dest, ppc_inst(PPC_RAW_NOP()));

	return 0;
}

void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
{
	struct fixup_entry *fcur, *fend;

	fcur = fixup_start;
	fend = fixup_end;

	for (; fcur < fend; fcur++) {
		if (patch_feature_section(value, fcur)) {
			WARN_ON(1);
			printk("Unable to patch feature section at %p - %p" \
				" with %p - %p\n",
				calc_addr(fcur, fcur->start_off),
				calc_addr(fcur, fcur->end_off),
				calc_addr(fcur, fcur->alt_start_off),
				calc_addr(fcur, fcur->alt_end_off));
		}
	}
}

#ifdef CONFIG_PPC_BOOK3S_64
static void do_stf_entry_barrier_fixups(enum stf_barrier_type types)
{
	unsigned int instrs[3], *dest;
	long *start, *end;
	int i;

	start = PTRRELOC(&__start___stf_entry_barrier_fixup);
	end = PTRRELOC(&__stop___stf_entry_barrier_fixup);

	instrs[0] = PPC_RAW_NOP();
	instrs[1] = PPC_RAW_NOP();
	instrs[2] = PPC_RAW_NOP();

	i = 0;
	if (types & STF_BARRIER_FALLBACK) {
		instrs[i++] = PPC_RAW_MFLR(_R10);
		instrs[i++] = PPC_RAW_NOP(); /* branch patched below */
		instrs[i++] = PPC_RAW_MTLR(_R10);
	} else if (types & STF_BARRIER_EIEIO) {
		instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */
	} else if (types & STF_BARRIER_SYNC_ORI) {
		instrs[i++] = PPC_RAW_SYNC();
		instrs[i++] = PPC_RAW_LD(_R10, _R13, 0);
		instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */
	}

	for (i = 0; start < end; start++, i++) {
		dest = (void *)start + *start;

		pr_devel("patching dest %lx\n", (unsigned long)dest);

		// See comment in do_entry_flush_fixups() RE order of patching
		if (types & STF_BARRIER_FALLBACK) {
			patch_instruction(dest, ppc_inst(instrs[0]));
			patch_instruction(dest + 2, ppc_inst(instrs[2]));
			patch_branch(dest + 1,
				     (unsigned long)&stf_barrier_fallback