path: root/arch/powerpc/kernel/head_book3s_32.S

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *  PowerPC version
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
 *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
 *  Adapted for Power Macintosh by Paul Mackerras.
 *  Low-level exception handlers and MMU support
 *  rewritten by Paul Mackerras.
 *    Copyright (C) 1996 Paul Mackerras.
 *  MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
 *
 *  This file contains the low-level support and setup for the
 *  PowerPC platform, including trap and interrupt dispatch.
 *  (The PPC 8xx embedded CPUs use head_8xx.S instead.)
 */

#include <linux/init.h>
#include <linux/pgtable.h>
#include <linux/linkage.h>

#include <asm/reg.h>
#include <asm/page.h>
#include <asm/mmu.h>
#include <asm/cputable.h>
#include <asm/cache.h>
#include <asm/thread_info.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/ptrace.h>
#include <asm/bug.h>
#include <asm/kvm_book3s_asm.h>
#include <asm/feature-fixups.h>
#include <asm/interrupt.h>

#include "head_32.h"

#define LOAD_BAT(n, reg, RA, RB)	\
	/* see the comment for clear_bats() -- Cort */ \
	li	RA,0;			\
	mtspr	SPRN_IBAT##n##U,RA;	\
	mtspr	SPRN_DBAT##n##U,RA;	\
	lwz	RA,(n*16)+0(reg);	\
	lwz	RB,(n*16)+4(reg);	\
	mtspr	SPRN_IBAT##n##U,RA;	\
	mtspr	SPRN_IBAT##n##L,RB;	\
	lwz	RA,(n*16)+8(reg);	\
	lwz	RB,(n*16)+12(reg);	\
	mtspr	SPRN_DBAT##n##U,RA;	\
	mtspr	SPRN_DBAT##n##L,RB

	__HEAD
_GLOBAL(_stext);

/*
 * _start is defined this way because the XCOFF loader in the OpenFirmware
 * on the powermac expects the entry point to be a procedure descriptor.
 */
_GLOBAL(_start);
	/*
	 * These are here for legacy reasons, the kernel used to
	 * need to look like a coff function entry for the pmac
	 * but we're always started by some kind of bootloader now.
	 *  -- Cort
	 */
	nop	/* used by __secondary_hold on prep (mtx) and chrp smp */
	nop	/* used by __secondary_hold on prep (mtx) and chrp smp */
	nop

/* PMAC
 * Enter here with the kernel text, data and bss loaded starting at
 * 0, running with virtual == physical mapping.
 * r5 points to the prom entry point (the client interface handler
 * address).  Address translation is turned on, with the prom
 * managing the hash table.  Interrupts are disabled.  The stack
 * pointer (r1) points to just below the end of the half-meg region
 * from 0x380000 - 0x400000, which is mapped in already.
 *
 * If we are booted from MacOS via BootX, we enter with the kernel
 * image loaded somewhere, and the following values in registers:
 *  r3: 'BooX' (0x426f6f58)
 *  r4: virtual address of boot_infos_t
 *  r5: 0
 *
 * PREP
 * This is jumped to on prep systems right after the kernel is relocated
 * to its proper place in memory by the boot loader.  The expected layout
 * of the regs is:
 *   r3: ptr to residual data
 *   r4: initrd_start or if no initrd then 0
 *   r5: initrd_end - unused if r4 is 0
 *   r6: Start of command line string
 *   r7: End of command line string
 *
 * This just gets a minimal mmu environment setup so we can call
 * start_here() to do the real work.
 * -- Cort
 */

	.globl	__start
__start:
/*
 * We have to do any OF calls before we map ourselves to KERNELBASE,
 * because OF may have I/O devices mapped into that area
 * (particularly on CHRP).
 */
	cmpwi	0,r5,0
	beq	1f

#ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE
	/* find out where we are now */
	bcl	20,31,$+4
0:	mflr	r8			/* r8 = runtime addr here */
	addis	r8,r8,(_stext - 0b)@ha
	addi	r8,r8,(_stext - 0b)@l	/* current runtime base addr */
	bl	prom_init
#endif /* CONFIG_PPC_OF_BOOT_TRAMPOLINE */

	/* We never return. We also hit that trap if trying to boot
	 * from OF while CONFIG_PPC_OF_BOOT_TRAMPOLINE isn't selected */
	trap

/*
 * Check for BootX signature when supporting PowerMac and branch to
 * appropriate trampoline if it's present
 */
#ifdef CONFIG_PPC_PMAC
1:	lis	r31,0x426f
	ori	r31,r31,0x6f58
	cmpw	0,r3,r31
	bne	1f
	bl	bootx_init
	trap
#endif /* CONFIG_PPC_PMAC */

1:	mr	r31,r3			/* save device tree ptr */
	li	r24,0			/* cpu # */

/*
 * early_init() does the early machine identification and does
 * the necessary low-level setup and clears the BSS
 *  -- Cort <cort@fsmlabs.com>
 */
	bl	early_init

/* Switch MMU off, clear BATs and flush TLB. At this point, r3 contains
 * the physical address we are running at, returned by early_init()
 */
 	bl	mmu_off
__after_mmu_off:
	bl	clear_bats
	bl	flush_tlbs

	bl	initial_bats
	bl	load_segment_registers
	bl	reloc_offset
	bl	early_hash_table
#if defined(CONFIG_BOOTX_TEXT)
	bl	setup_disp_bat
#endif
#ifdef CONFIG_PPC_EARLY_DEBUG_CPM
	bl	setup_cpm_bat
#endif
#ifdef CONFIG_PPC_EARLY_DEBUG_USBGECKO
	bl	setup_usbgecko_bat
#endif

/*
 * Call setup_cpu for CPU 0 and initialize 6xx Idle
 */
	bl	reloc_offset
	li	r24,0			/* cpu# */
	bl	call_setup_cpu		/* Call setup_cpu for this CPU */
	bl	reloc_offset
	bl	init_idle_6xx


/*
 * We need to run with _start at physical address 0.
 * On CHRP, we are loaded at 0x10000 since OF on CHRP uses
 * the exception vectors at 0 (and therefore this copy
 * overwrites OF's exception vectors with our own).
 * The MMU is off at this point.
 */
	bl	reloc_offset
	mr	r26,r3
	addis	r4,r3,KERNELBASE@h	/* current address of _start */
	lis	r5,PHYSICAL_START@h
	cmplw	0,r4,r5			/* already running at PHYSICAL_START? */
	bne	relocate_kernel
/*
 * we now have the 1st 16M of ram mapped with the bats.
 * prep needs the mmu to be turned on here, but pmac already has it on.
 * this shouldn't bother the pmac since it just gets turned on again
 * as we jump to our code at KERNELBASE. -- Cort
 * Actually no, pmac doesn't have it on any more. BootX enters with MMU
 * off, and in other cases, we now turn it off before changing BATs above.
 */
turn_on_mmu:
	mfmsr	r0
	ori	r0,r0,MSR_DR|MSR_IR|MSR_RI
	mtspr	SPRN_SRR1,r0
	lis	r0,start_here@h
	ori	r0,r0,start_here@l
	mtspr	SPRN_SRR0,r0
	rfi				/* enables MMU */

/*
 * We need __secondary_hold as a place to hold the other cpus on
 * an SMP machine, even when we are running a UP kernel.
 */
	. = 0xc0			/* for prep bootloader */
	li	r3,1			/* MTX only has 1 cpu */
	.globl	__secondary_hold
__secondary_hold:
	/* tell the master we're here */
	stw	r3,__secondary_hold_acknowledge@l(0)
#ifdef CONFIG_SMP
100:	lwz	r4,0(0)
	/* wait until we're told to start */
	cmpw	0,r4,r3
	bne	100b
	/* our cpu # was at addr 0 - go */
	mr	r24,r3			/* cpu # */
	b	__secondary_start
#else
	b	.
#endif /* CONFIG_SMP */

	.globl	__secondary_hold_spinloop
__secondary_hold_spinloop:
	.long	0
	.globl	__secondary_hold_acknowledge
__secondary_hold_acknowledge:
	.long	-1

/* System reset */
/* core99 pmac starts the seconary here by changing the vector, and
   putting it back to what it was (unknown_async_exception) when done.  */
	EXCEPTION(INTERRUPT_SYSTEM_RESET, Reset, unknown_async_exception)

/* Machine check */
/*
 * On CHRP, this is complicated by the fact that we could get a
 * machine check inside RTAS, and we have no guarantee that certain
 * critical registers will have the values we expect.  The set of
 * registers that might have bad values includes all the GPRs
 * and all the BATs.  We indicate that we are in RTAS by putting
 * a non-zero value, the address of the exception frame to use,
 * in thread.rtas_sp.  The machine check handler checks thread.rtas_sp
 * and uses its value if it is non-zero.
 * (Other exception handlers assume that r1 is a valid kernel stack
 * pointer when we take an exception from supervisor mode.)
 *	-- paulus.
 */
	START_EXCEPTION(INTERRUPT_MACHINE_CHECK, MachineCheck)
	EXCEPTION_PROLOG_0
#ifdef CONFIG_PPC_CHRP
	mtspr	SPRN_SPRG_SCRATCH2,r1
	mfspr	r1, SPRN_SPRG_THREAD
	lwz	r1, RTAS_SP(r1)
	cmpwi	cr1, r1, 0
	bne	cr1, 7f
	mfspr	r1, SPRN_SPRG_SCRATCH2
#endif /* CONFIG_PPC_CHRP */
	EXCEPTION_PROLOG_1
7:	EXCEPTION_PROLOG_2 0x200 MachineCheck
#ifdef CONFIG_PPC_CHRP
	beq	cr1, 1f
	twi	31, 0, 0
#endif
1:	prepare_transfer_to_handler
	bl	machine_check_exception
	b	interrupt_return

/* Data access exception. */
	START_EXCEPTION(INTERRUPT_DATA_STORAGE, DataAccess)
#ifdef CONFIG_PPC_BOOK3S_604
BEGIN_MMU_FTR_SECTION
	mtspr	SPRN_SPRG_SCRATCH2,r10
	mfspr	r10, SPRN_SPRG_THREAD
	stw	r11, THR11(r10)
	mfspr	r10, SPRN_DSISR
	mfcr	r11
	andis.	r10, r10, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h
	mfspr	r10, SPRN_SPRG_THREAD
	beq	hash_page_dsi
.Lhash_page_dsi_cont:
	mtcr	r11
	lwz	r11, THR11(r10)
	mfspr	r10, SPRN_SPRG_SCRATCH2
MMU_FTR_SECTION_ELSE
	b	1f
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
#endif
1:	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
	EXCEPTION_PROLOG_1
	EXCEPTION_PROLOG_2 INTERRUPT_DATA_STORAGE DataAccess handle_dar_dsisr=1
	prepare_transfer_to_handler
	lwz	r5, _DSISR(r1)
	andis.	r0, r5, DSISR_DABRMATCH@h
	bne-	1f
	bl	do_page_fault
	b	interrupt_return
1:	bl	do_break
	REST_NVGPRS(r1)
	b	interrupt_return


/* Instruction access exception. */
	START_EXCEPTION(INTERRUPT_INST_STORAGE, InstructionAccess)
	mtspr	SPRN_SPRG_SCRATCH0,r10
	mtspr	SPRN_SPRG_SCRATCH1,r11
	mfspr	r10, SPRN_SPRG_THREAD
	mfspr	r11, SPRN_SRR0
	stw	r11, SRR0(r10)
	mfspr	r11, SPRN_SRR1		/* check whether user or kernel */
	stw	r11, SRR1(r10)
	mfcr	r10
#ifdef CONFIG_PPC_BOOK3S_604
BEGIN_MMU_FTR_SECTION
	andis.	r11, r11, SRR1_ISI_NOPT@h	/* no pte found? */
	bne	hash_page_isi
.Lhash_page_isi_cont:
	mfspr	r11, SPRN_SRR1		/* check whether user or kernel */
END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
#endif
	andi.	r11, r11, MSR_PR

	EXCEPTION_PROLOG_1
	EXCEPTION_PROLOG_2 INTERRUPT_INST_STORAGE InstructionAccess
	andis.	r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
	stw	r5, _DSISR(r11)
	stw	r12, _DAR(r11)
	prepare_transfer_to_handler
	bl	do_page_fault
	b	interrupt_return

/* External interrupt */
	EXCEPTION(INTERRUPT_EXTERNAL, HardwareInterrupt, do_IRQ)

/* Alignment exception */
	START_EXCEPTION(INTERRUPT_ALIGNMENT, Alignment)
	EXCEPTION_PROLOG INTERRUPT_ALIGNMENT Alignment handle_dar_dsisr=1
	prepare_transfer_to_handler
	bl	alignment_exception
	REST_NVGPRS(r1)
	b	interrupt_return

/* Program check exception */
	START_EXCEPTION(INTERRUPT_PROGRAM, ProgramCheck)
	EXCEPTION_PROLOG INTERRUPT_PROGRAM ProgramCheck
	prepare_transfer_to_handler
	bl	program_check_exception
	REST_NVGPRS(r1)
	b	interrupt_return

/* Floating-point unavailable */
	START_EXCEPTION(0x800, FPUnavailable)
#ifdef CONFIG_PPC_FPU
BEGIN_FTR_SECTION
/*
 * Certain Freescale cores don't have a FPU and treat fp instructions
 * as a FP Unavailable exception.  Redirect to illegal/emulation handling.
 */
	b 	ProgramCheck
END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
	EXCEPTION_PROLOG INTERRUPT_FP_UNAVAIL FPUnavailable
	beq	1f
	bl	load_up_fpu		/* if from user, just load it up */
	b	fast_exception_return
1:	prepare_transfer_to_handler
	bl	kernel_fp_unavailable_exception
	b	interrupt_return
#else
	b 	ProgramCheck
#endif

/* Decrementer */
	EXCEPTION(INTERRUPT_DECREMENTER, Decrementer, timer_interrupt)

	EXCEPTION(0xa00, Trap_0a, unknown_exception)
	EXCEPTION(0xb00, Trap_0b, unknown_exception)

/* System call */
	START_EXCEPTION(INTERRUPT_SYSCALL, SystemCall)
	SYSCALL_ENTRY	INTERRUPT_SYSCALL

	EXCEPTION(INTERRUPT_TRACE, SingleStep, single_step_exception)
	EXCEPTION(0xe00, Trap_0e, unknown_exception)

/*
 * The Altivec unavailable trap is at 0x0f20.  Foo.
 * We effectively remap it to 0x3000.
 * We include an altivec unavailable exception vector even if
 * not configured for Altivec, so that you can't panic a
 * non-altivec kernel running on a machine with altivec just
 * by executing an altivec instruction.
 */
	START_EXCEPTION(INTERRUPT_PERFMON, PerformanceMonitorTrap)
	b	PerformanceMonitor

	START_EXCEPTION(INTERRUPT_ALTIVEC_UNAVAIL, AltiVecUnavailableTrap)
	b	AltiVecUnavailable

	__HEAD
/*
 * Handle TLB miss for instruction on 603/603e.
 * Note: we get an alternate set of r0 - r3 to use automatically.
 */
	. = INTERRUPT_INST_TLB_MISS_603
InstructionTLBMiss:
	/* Get PTE (linux-style) and check access */
	mfspr	r0,SPRN_IMISS
	mfspr	r2, SPRN_SDR1
	li	r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
	rlwinm	r2, r2, 28, 0xfffff000
	rlwimi	r2,r0,12,20,29		/* insert top 10 bits of address */
	lwz	r2,0(r2)		/* get pmd entry */
#ifdef CONFIG_EXECMEM
	rlwinm	r3, r0, 4, 0xf
	subi	r3, r3, NUM_USER_SEGMENTS
#endif
	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */
	beq-	InstructionAddressInvalid	/* return if no mapping */
	rlwimi	r2,r0,22,20,29		/* insert next 10 bits of address */
	lwz	r2,0(r2)		/* get linux-style pte */
	andc.	r1,r1,r2		/* check access & ~permission */
	bne-	InstructionAddressInvalid /* return if access not permitted */
	/* Convert linux-style PTE to low word of PPC-style PTE */
#ifdef CONFIG_EXECMEM
	rlwimi	r2, r3, 1, 31, 31	/* userspace ? -> PP lsb */
#endif
	ori	r1, r1, 0xe06		/* clear out reserved bits */
	andc	r1, r2, r1		/* PP = user? 1 : 0 */
BEGIN_FTR_SECTION
	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */
END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
	mtspr	SPRN_RPA,r1
	tlbli	r0
	mfspr	r3,SPRN_SRR1		/* Need to restore CR0 */
	mtcrf	0x80,r3
	rfi
InstructionAddressInvalid:
	mfspr	r3,SPRN_SRR1
	rlwinm	r1,r3,9,6,6	/* Get load/store bit */

	addis	r1,r1,0x2000
	mtspr	SPRN_DSISR,r1	/* (shouldn't be needed) */
	andi.	r2,r3,0xFFFF	/* Clear upper bits of SRR1 */
	or	r2,r2,r1
	mtspr	SPRN_SRR1,r2
	mfspr	r1,SPRN_IMISS	/* Get failing address */
	rlwinm.	r2,r2,0,31,31	/* Check for little endian access */
	rlwimi	r2,r2,1,30,30	/* change 1 -> 3 */
	xor	r1,r1,r2
	mtspr	SPRN_DAR,r1	/* Set fault address */
	mfmsr	r0		/* Restore "normal" registers */
	xoris	r0,r0,MSR_TGPR>>16
	mtcrf	0x80,r3		/* Restore CR0 */
	mtmsr	r0
	b	InstructionAccess

/*
 * Handle TLB miss for DATA Load operation on 603/603e
 */
	. = INTERRUPT_DATA_LOAD_TLB_MISS_603
DataLoadTLBMiss:
	/* Get PTE (linux-style) and check access */
	mfspr	r0,SPRN_DMISS
	mfspr	r2, SPRN_SDR1
	rlwinm	r1, r2, 28, 0xfffff000
	rlwimi	r1,r0,12,20,29		/* insert top 10 bits of address */
	lwz	r2,0(r1)		/* get pmd entry */
	rlwinm	r3, r0, 4, 0xf
	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */
	subi	r3, r3, NUM_USER_SEGMENTS
	beq-	2f			/* bail if no mapping */
1:	rlwimi	r2,r0,22,20,29		/* insert next 10 bits of address */
	lwz	r2,0(r2)		/* get linux-style pte */
	li	r1, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_READ
	andc.	r1,r1,r2		/* check access & ~permission */
	bne-	DataAddressInvalid	/* return if access not permitted */
	/* Convert linux-style PTE to low word of PPC-style PTE */
	rlwinm	r1,r2,32-9,30,30	/* _PAGE_WRITE -> PP msb */
	rlwimi	r2,r3,2,30,31		/* userspace ? -> PP */
	rlwimi	r1,r2,32-3,24,24	/* _PAGE_WRITE -> _PAGE_DIRTY */
	xori	r1,r1,_PAGE_DIRTY	/* clear dirty when not rw */
	ori	r1,r1,0xe04		/* clear out reserved bits */
	andc	r1,r2,r1		/* PP = user? rw? 1: 3: 0 */
BEGIN_FTR_SECTION
	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */
END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
	mtspr	SPRN_RPA,r1
BEGIN_MMU_FTR_SECTION
	li	r3,1
	mfspr	r1,SPRN_SPRG_603_LRU
	rlwinm	r2,r0,20,27,31		/* Get Address bits 15:19 */
	slw	r3,r3,r2
	xor	r1,r3,r1
	srw	r3,r1,r2
	mtspr   SPRN_SPRG_603_LRU,r1
	mfspr	r2,SPRN_SRR1
	rlwimi	r2,r3,31-14,14,14
	mtspr   SPRN_SRR1,r2
	mtcrf	0x80,r2
	tlbld	r0
	rfi
MMU_FTR_SECTION_ELSE
	mfspr	r2,SPRN_SRR1		/* Need to restore CR0 */
	mtcrf	0x80,r2
	tlbld	r0
	rfi
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)

2:	lis     r2, (swapper_pg_dir - PAGE_OFFSET)@ha
	addi    r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l        /* kernel page table */
	rlwimi	r2,r0,12,20,29		/* insert top 10 bits of address */
	lwz	r2,0(r2)		/* get pmd entry */
	cmpwi	cr0,r2,0
	beq-	DataAddressInvalid	/* return if no mapping */
	stw	r2,0(r1)
	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */
	b	1b
DataAddressInvalid:
	mfspr	r3,SPRN_SRR1
	rlwinm	r1,r3,9,6,6	/* Get load/store bit */
	addis	r1,r1,0x2000
	mtspr	SPRN_DSISR,r1
	andi.	r2,r3,0xFFFF	/* Clear upper bits of SRR1 */
	mtspr	SPRN_SRR1,r2
	mfspr	r1,SPRN_DMISS	/* Get failing address */
	rlwinm.	r2,r2,0,31,31	/* Check for little endian access */
	beq	20f		/* Jump if big endian */
	xori	r1,r1,3
20:	mtspr	SPRN_DAR,r1	/* Set fault address */
	mfmsr	r0		/* Restore "normal" registers */
	xoris	r0,r0,MSR_TGPR>>16
	mtcrf	0x80,r3		/* Restore CR0 */
	mtmsr	r0
	b	DataAccess

/*
 * Handle TLB miss for DATA Store on 603/603e
 */
	. = INTERRUPT_DATA_STORE_TLB_MISS_603
DataStoreTLBMiss:
	/* Get PTE (linux-style) and check access */
	mfspr	r0,SPRN_DMISS
	mfspr	r2, SPRN_SDR1
	rlwinm	r1, r2, 28, 0xfffff000
	rlwimi	r1,r0,12,20,29		/* insert top 10 bits of address */
	lwz	r2,0(r1)		/* get pmd entry */
	rlwinm	r3, r0, 4, 0xf
	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */
	subi	r3, r3, NUM_USER_SEGMENTS
	beq-	2f			/* bail if no mapping */
1:
	rlwimi	r2,r0,22,20,29		/* insert next 10 bits of address */
	lwz	r2,0(r2)		/* get linux-style pte */
	li	r1, _PAGE_RW | _PAGE_DIRTY | _PAGE_PRESENT | _PAGE_ACCESSED
	andc.	r1,r1,r2		/* check access & ~permission */
	bne-	DataAddressInvalid	/* return if access not permitted */
	/* Convert linux-style PTE to low word of PPC-style PTE */
	rlwimi	r2,r3,1,31,31		/* userspace ? -> PP lsb */
	li	r1,0xe06		/* clear out reserved bits & PP msb */
	andc	r1,r2,r1		/* PP = user? 1: 0 */
BEGIN_FTR_SECTION
	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */
END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
	mtspr	SPRN_RPA,r1
	mfspr	r2,SPRN_SRR1		/* Need to restore CR0 */
	mtcrf	0x80,r2
BEGIN_MMU_FTR_SECTION
	li	r3,1
	mfspr	r1,SPRN_SPRG_603_LRU
	rlwinm	r2,r0,20,27,31		/* Get Address bits 15:19 */
	slw	r3,r3,r2
	xor	r1,r3,r1
	srw	r3,r1,r2
	mtspr   SPRN_SPRG_603_LRU,r1
	mfspr	r2,SPRN_SRR1
	rlwimi	r2,r3,31-14,14,14
	mtspr   SPRN_SRR1,r2
	mtcrf	0x80,r2
	tlbld	r0
	rfi
MMU_FTR_SECTION_ELSE
	mfspr	r2,SPRN_SRR1		/* Need to restore CR0 */
	mtcrf	0x80,r2
	tlbld	r0
	rfi
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)

2:	lis     r2, (swapper_pg_dir - PAGE_OFFSET)@ha
	addi    r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l        /* kernel page table */
	rlwimi	r2,r0,12,20,29		/* insert top 10 bits of address */
	lwz	r2,0(r2)		/* get pmd entry */
	cmpwi	cr0,r2,0
	beq-	DataAddressInvalid	/* return if no mapping */
	stw	r2,0(r1)
	rlwinm	r2,r2,0,0,19		/* extract address of pte page */
	b	1b

#ifndef CONFIG_ALTIVEC
#define altivec_assist_exception	unknown_exception
#endif

#ifndef CONFIG_TAU_INT
#define TAUException	unknown_async_exception
#endif

	EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception)
	EXCEPTION(0x1400, SMI, SMIException)
	EXCEPTION(0x1500, Trap_15, unknown_exception)
	EXCEPTION(0x1600, Trap_16, altivec_assist_exception)
	EXCEPTION(0x1700, Trap_17, TAUException)
	EXCEPTION(0x1800, Trap_18, unknown_exception)
	EXCEPTION(0x1900, Trap_19, unknown_exception)
	EXCEPTION(0x1a00, Trap_1a, unknown_exception)
	EXCEPTION(0x1b00, Trap_1b, unknown_exception)
	EXCEPTION(0x1c00, Trap_1c, unknown_exception)
	EXCEPTION(0x1d00, Trap_1d, unknown_exception)
	EXCEPTION(0x1e00, Trap_1e, unknown_exception)
	EXCEPTION(0x1f00, Trap_1f, unknown_exception)
	EXCEPTION(0x2000, RunMode, RunModeException)
	EXCEPTION(0x2100, Trap_21, unknown_exception)
	EXCEPTION(0x2200, Trap_22, unknown_exception)
	EXCEPTION(0x2300, Trap_23, unknown_exception)
	EXCEPTION(0x2400, Trap_24, unknown_exception)
	EXCEPTION(0x2500, Trap_25, unknown_exception)
	EXCEPTION(0x2600, Trap_2