path: root/arch/powerpc/kernel/exceptions-64e.S

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *  Boot code and exception vectors for Book3E processors
 *
 *  Copyright (C) 2007 Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp.
 */

#include <linux/linkage.h>
#include <linux/threads.h>
#include <asm/reg.h>
#include <asm/page.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/cputable.h>
#include <asm/setup.h>
#include <asm/thread_info.h>
#include <asm/exception-64e.h>
#include <asm/bug.h>
#include <asm/irqflags.h>
#include <asm/ptrace.h>
#include <asm/ppc-opcode.h>
#include <asm/mmu.h>
#include <asm/hw_irq.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_booke_hv_asm.h>
#include <asm/feature-fixups.h>
#include <asm/context_tracking.h>

/* 64e interrupt returns always use SRR registers */
#define fast_interrupt_return fast_interrupt_return_srr
#define interrupt_return interrupt_return_srr

/* XXX This will ultimately add space for a special exception save
 *     structure used to save things like SRR0/SRR1, SPRGs, MAS, etc...
 *     when taking special interrupts. For now we don't support that,
 *     special interrupts from within a non-standard level will probably
 *     blow you up
 */
#define SPECIAL_EXC_SRR0	0
#define SPECIAL_EXC_SRR1	1
#define SPECIAL_EXC_SPRG_GEN	2
#define SPECIAL_EXC_SPRG_TLB	3
#define SPECIAL_EXC_MAS0	4
#define SPECIAL_EXC_MAS1	5
#define SPECIAL_EXC_MAS2	6
#define SPECIAL_EXC_MAS3	7
#define SPECIAL_EXC_MAS6	8
#define SPECIAL_EXC_MAS7	9
#define SPECIAL_EXC_MAS5	10	/* E.HV only */
#define SPECIAL_EXC_MAS8	11	/* E.HV only */
#define SPECIAL_EXC_IRQHAPPENED	12
#define SPECIAL_EXC_DEAR	13
#define SPECIAL_EXC_ESR		14
#define SPECIAL_EXC_SOFTE	15
#define SPECIAL_EXC_CSRR0	16
#define SPECIAL_EXC_CSRR1	17
/* must be even to keep 16-byte stack alignment */
#define SPECIAL_EXC_END		18

#define SPECIAL_EXC_FRAME_SIZE	(INT_FRAME_SIZE + SPECIAL_EXC_END * 8)
#define SPECIAL_EXC_FRAME_OFFS  (INT_FRAME_SIZE - 288)

#define SPECIAL_EXC_STORE(reg, name) \
	std	reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)

#define SPECIAL_EXC_LOAD(reg, name) \
	ld	reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)

SYM_CODE_START_LOCAL(special_reg_save)
	/*
	 * We only need (or have stack space) to save this stuff if
	 * we interrupted the kernel.
	 */
	ld	r3,_MSR(r1)
	andi.	r3,r3,MSR_PR
	bnelr

	/*
	 * Advance to the next TLB exception frame for handler
	 * types that don't do it automatically.
	 */
	LOAD_REG_ADDR(r11,extlb_level_exc)
	lwz	r12,0(r11)
	mfspr	r10,SPRN_SPRG_TLB_EXFRAME
	add	r10,r10,r12
	mtspr	SPRN_SPRG_TLB_EXFRAME,r10

	/*
	 * Save registers needed to allow nesting of certain exceptions
	 * (such as TLB misses) inside special exception levels
	 */
	mfspr	r10,SPRN_SRR0
	SPECIAL_EXC_STORE(r10,SRR0)
	mfspr	r10,SPRN_SRR1
	SPECIAL_EXC_STORE(r10,SRR1)
	mfspr	r10,SPRN_SPRG_GEN_SCRATCH
	SPECIAL_EXC_STORE(r10,SPRG_GEN)
	mfspr	r10,SPRN_SPRG_TLB_SCRATCH
	SPECIAL_EXC_STORE(r10,SPRG_TLB)
	mfspr	r10,SPRN_MAS0
	SPECIAL_EXC_STORE(r10,MAS0)
	mfspr	r10,SPRN_MAS1
	SPECIAL_EXC_STORE(r10,MAS1)
	mfspr	r10,SPRN_MAS2
	SPECIAL_EXC_STORE(r10,MAS2)
	mfspr	r10,SPRN_MAS3
	SPECIAL_EXC_STORE(r10,MAS3)
	mfspr	r10,SPRN_MAS6
	SPECIAL_EXC_STORE(r10,MAS6)
	mfspr	r10,SPRN_MAS7
	SPECIAL_EXC_STORE(r10,MAS7)
BEGIN_FTR_SECTION
	mfspr	r10,SPRN_MAS5
	SPECIAL_EXC_STORE(r10,MAS5)
	mfspr	r10,SPRN_MAS8
	SPECIAL_EXC_STORE(r10,MAS8)

	/* MAS5/8 could have inappropriate values if we interrupted KVM code */
	li	r10,0
	mtspr	SPRN_MAS5,r10
	mtspr	SPRN_MAS8,r10
END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
	mfspr	r10,SPRN_DEAR
	SPECIAL_EXC_STORE(r10,DEAR)
	mfspr	r10,SPRN_ESR
	SPECIAL_EXC_STORE(r10,ESR)

	ld	r10,_NIP(r1)
	SPECIAL_EXC_STORE(r10,CSRR0)
	ld	r10,_MSR(r1)
	SPECIAL_EXC_STORE(r10,CSRR1)

	blr
SYM_CODE_END(special_reg_save)

SYM_CODE_START_LOCAL(ret_from_level_except)
	ld	r3,_MSR(r1)
	andi.	r3,r3,MSR_PR
	beq	1f
	REST_NVGPRS(r1)
	b	interrupt_return
1:

	LOAD_REG_ADDR(r11,extlb_level_exc)
	lwz	r12,0(r11)
	mfspr	r10,SPRN_SPRG_TLB_EXFRAME
	sub	r10,r10,r12
	mtspr	SPRN_SPRG_TLB_EXFRAME,r10

	/*
	 * It's possible that the special level exception interrupted a
	 * TLB miss handler, and inserted the same entry that the
	 * interrupted handler was about to insert.  On CPUs without TLB
	 * write conditional, this can result in a duplicate TLB entry.
	 * Wipe all non-bolted entries to be safe.
	 *
	 * Note that this doesn't protect against any TLB misses
	 * we may take accessing the stack from here to the end of
	 * the special level exception.  It's not clear how we can
	 * reasonably protect against that, but only CPUs with
	 * neither TLB write conditional nor bolted kernel memory
	 * are affected.  Do any such CPUs even exist?
	 */
	PPC_TLBILX_ALL(0,R0)

	REST_NVGPRS(r1)

	SPECIAL_EXC_LOAD(r10,SRR0)
	mtspr	SPRN_SRR0,r10
	SPECIAL_EXC_LOAD(r10,SRR1)
	mtspr	SPRN_SRR1,r10
	SPECIAL_EXC_LOAD(r10,SPRG_GEN)
	mtspr	SPRN_SPRG_GEN_SCRATCH,r10
	SPECIAL_EXC_LOAD(r10,SPRG_TLB)
	mtspr	SPRN_SPRG_TLB_SCRATCH,r10
	SPECIAL_EXC_LOAD(r10,MAS0)
	mtspr	SPRN_MAS0,r10
	SPECIAL_EXC_LOAD(r10,MAS1)
	mtspr	SPRN_MAS1,r10
	SPECIAL_EXC_LOAD(r10,MAS2)
	mtspr	SPRN_MAS2,r10
	SPECIAL_EXC_LOAD(r10,MAS3)
	mtspr	SPRN_MAS3,r10
	SPECIAL_EXC_LOAD(r10,MAS6)
	mtspr	SPRN_MAS6,r10
	SPECIAL_EXC_LOAD(r10,MAS7)
	mtspr	SPRN_MAS7,r10
BEGIN_FTR_SECTION
	SPECIAL_EXC_LOAD(r10,MAS5)
	mtspr	SPRN_MAS5,r10
	SPECIAL_EXC_LOAD(r10,MAS8)
	mtspr	SPRN_MAS8,r10
END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)

	SPECIAL_EXC_LOAD(r10,DEAR)
	mtspr	SPRN_DEAR,r10
	SPECIAL_EXC_LOAD(r10,ESR)
	mtspr	SPRN_ESR,r10

	stdcx.	r0,0,r1		/* to clear the reservation */

	REST_GPRS(2, 9, r1)

	ld	r10,_CTR(r1)
	ld	r11,_XER(r1)
	mtctr	r10
	mtxer	r11

	blr
SYM_CODE_END(ret_from_level_except)

.macro ret_from_level srr0 srr1 paca_ex scratch
	bl	ret_from_level_except

	ld	r10,_LINK(r1)
	ld	r11,_CCR(r1)
	ld	r0,GPR13(r1)
	mtlr	r10
	mtcr	r11

	REST_GPRS(10, 12, r1)
	mtspr	\scratch,r0

	std	r10,\paca_ex+EX_R10(r13);
	std	r11,\paca_ex+EX_R11(r13);
	ld	r10,_NIP(r1)
	ld	r11,_MSR(r1)
	REST_GPR(0, r1)
	REST_GPR(1, r1)
	mtspr	\srr0,r10
	mtspr	\srr1,r11
	ld	r10,\paca_ex+EX_R10(r13)
	ld	r11,\paca_ex+EX_R11(r13)
	mfspr	r13,\scratch
.endm

SYM_CODE_START_LOCAL(ret_from_crit_except)
	ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH
	rfci
SYM_CODE_END(ret_from_crit_except)

SYM_CODE_START_LOCAL(ret_from_mc_except)
	ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH
	rfmci
SYM_CODE_END(ret_from_mc_except)

/* Exception prolog code for all exceptions */
#define EXCEPTION_PROLOG(n, intnum, type, addition)	    		    \
	mtspr	SPRN_SPRG_##type##_SCRATCH,r13;	/* get spare registers */   \
	mfspr	r13,SPRN_SPRG_PACA;	/* get PACA */			    \
	std	r10,PACA_EX##type+EX_R10(r13);				    \
	std	r11,PACA_EX##type+EX_R11(r13);				    \
	m