diff options
69 files changed, 2138 insertions, 1492 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index cedfeb9ab5a3..3805de65c6ac 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7839,7 +7839,6 @@ F: Documentation/admin-guide/efi-stub.rst F: arch/*/include/asm/efi.h F: arch/*/kernel/efi.c F: arch/arm/boot/compressed/efi-header.S -F: arch/arm64/kernel/efi-entry.S F: arch/x86/platform/efi/ F: drivers/firmware/efi/ F: include/linux/efi*.h diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h index 4bdd930167c0..b95241b1ca65 100644 --- a/arch/arm/include/asm/efi.h +++ b/arch/arm/include/asm/efi.h @@ -43,9 +43,6 @@ void efi_virtmap_unload(void); /* arch specific definitions used by the stub code */ -struct screen_info *alloc_screen_info(void); -void free_screen_info(struct screen_info *si); - /* * A reasonable upper bound for the uncompressed kernel size is 32 MBytes, * so we will reserve that amount of memory. We have no easy way to tell what diff --git a/arch/arm/kernel/efi.c b/arch/arm/kernel/efi.c index e50ad7eefc02..882104f43b3b 100644 --- a/arch/arm/kernel/efi.c +++ b/arch/arm/kernel/efi.c @@ -75,38 +75,13 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) return 0; } -static unsigned long __initdata screen_info_table = EFI_INVALID_TABLE_ADDR; static unsigned long __initdata cpu_state_table = EFI_INVALID_TABLE_ADDR; const efi_config_table_type_t efi_arch_tables[] __initconst = { - {LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID, &screen_info_table}, {LINUX_EFI_ARM_CPU_STATE_TABLE_GUID, &cpu_state_table}, {} }; -static void __init load_screen_info_table(void) -{ - struct screen_info *si; - - if (screen_info_table != EFI_INVALID_TABLE_ADDR) { - si = early_memremap_ro(screen_info_table, sizeof(*si)); - if (!si) { - pr_err("Could not map screen_info config table\n"); - return; - } - screen_info = *si; - early_memunmap(si, sizeof(*si)); - - /* dummycon on ARM needs non-zero values for columns/lines */ - screen_info.orig_video_cols = 80; - screen_info.orig_video_lines = 25; - - if (memblock_is_map_memory(screen_info.lfb_base)) - memblock_mark_nomap(screen_info.lfb_base, - screen_info.lfb_size); - } -} - static void __init load_cpu_state_table(void) { if (cpu_state_table != EFI_INVALID_TABLE_ADDR) { @@ -145,7 +120,11 @@ void __init arm_efi_init(void) { efi_init(); - load_screen_info_table(); + if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI) { + /* dummycon on ARM needs non-zero values for columns/lines */ + screen_info.orig_video_cols = 80; + screen_info.orig_video_lines = 25; + } /* ARM does not permit early mappings to persist across paging_init() */ efi_memmap_unmap(); diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 439e2bc5d5d8..31d13a6001df 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -14,8 +14,16 @@ #ifdef CONFIG_EFI extern void efi_init(void); + +bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg); #else #define efi_init() + +static inline +bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg) +{ + return false; +} #endif int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); @@ -25,6 +33,7 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); ({ \ efi_virtmap_load(); \ __efi_fpsimd_begin(); \ + spin_lock(&efi_rt_lock); \ }) #undef arch_efi_call_virt @@ -33,10 +42,12 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); #define arch_efi_call_virt_teardown() \ ({ \ + spin_unlock(&efi_rt_lock); \ __efi_fpsimd_end(); \ efi_virtmap_unload(); \ }) +extern spinlock_t efi_rt_lock; efi_status_t __efi_rt_asm_wrapper(void *, const char *, ...); #define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT) @@ -76,13 +87,23 @@ static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr) return (image_addr & ~(SZ_1G - 1UL)) + (1UL << (VA_BITS_MIN - 1)); } -#define alloc_screen_info(x...) &screen_info - -static inline void free_screen_info(struct screen_info *si) +static inline unsigned long efi_get_kimg_min_align(void) { + extern bool efi_nokaslr; + + /* + * Although relocatable kernels can fix up the misalignment with + * respect to MIN_KIMG_ALIGN, the resulting virtual text addresses are + * subtly out of sync with those recorded in the vmlinux when kaslr is + * disabled but the image required relocation anyway. Therefore retain + * 2M alignment if KASLR was explicitly disabled, even if it was not + * going to be activated to begin with. + */ + return efi_nokaslr ? MIN_KIMG_ALIGN : EFI_KIMG_ALIGN; } #define EFI_ALLOC_ALIGN SZ_64K +#define EFI_ALLOC_LIMIT ((1UL << 48) - 1) /* * On ARM systems, virtually remapped UEFI runtime services are set up in two diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 8dd925f4a4c6..ceba6792f5b3 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -36,12 +36,6 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ syscall.o proton-pack.o idreg-override.o idle.o \ patching.o -targets += efi-entry.o - -OBJCOPYFLAGS := --prefix-symbols=__efistub_ -$(obj)/%.stub.o: $(obj)/%.o FORCE - $(call if_changed,objcopy) - obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ sys_compat.o obj-$(CONFIG_COMPAT) += sigreturn32.o @@ -57,8 +51,7 @@ obj-$(CONFIG_CPU_PM) += sleep.o suspend.o obj-$(CONFIG_CPU_IDLE) += cpuidle.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_KGDB) += kgdb.o -obj-$(CONFIG_EFI) += efi.o efi-entry.stub.o \ - efi-rt-wrapper.o +obj-$(CONFIG_EFI) += efi.o efi-rt-wrapper.o obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o obj-$(CONFIG_ACPI) += acpi.o diff --git a/arch/arm64/kernel/efi-entry.S b/arch/arm64/kernel/efi-entry.S deleted file mode 100644 index 61a87fa1c305..000000000000 --- a/arch/arm64/kernel/efi-entry.S +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * EFI entry point. - * - * Copyright (C) 2013, 2014 Red Hat, Inc. - * Author: Mark Salter <msalter@redhat.com> - */ -#include <linux/linkage.h> -#include <linux/init.h> - -#include <asm/assembler.h> - - __INIT - -SYM_CODE_START(efi_enter_kernel) - /* - * efi_pe_entry() will have copied the kernel image if necessary and we - * end up here with device tree address in x1 and the kernel entry - * point stored in x0. Save those values in registers which are - * callee preserved. - */ - ldr w2, =primary_entry_offset - add x19, x0, x2 // relocated Image entrypoint - mov x20, x1 // DTB address - - /* - * Clean the copied Image to the PoC, and ensure it is not shadowed by - * stale icache entries from before relocation. - */ - ldr w1, =kernel_size - add x1, x0, x1 - bl dcache_clean_poc - ic ialluis - - /* - * Clean the remainder of this routine to the PoC - * so that we can safely disable the MMU and caches. - */ - adr x0, 0f - adr x1, 3f - bl dcache_clean_poc -0: - /* Turn off Dcache and MMU */ - mrs x0, CurrentEL - cmp x0, #CurrentEL_EL2 - b.ne 1f - mrs x0, sctlr_el2 - bic x0, x0, #1 << 0 // clear SCTLR.M - bic x0, x0, #1 << 2 // clear SCTLR.C - pre_disable_mmu_workaround - msr sctlr_el2, x0 - isb - b 2f -1: - mrs x0, sctlr_el1 - bic x0, x0, #1 << 0 // clear SCTLR.M - bic x0, x0, #1 << 2 // clear SCTLR.C - pre_disable_mmu_workaround - msr sctlr_el1, x0 - isb -2: - /* Jump to kernel entry point */ - mov x0, x20 - mov x1, xzr - mov x2, xzr - mov x3, xzr - br x19 -3: -SYM_CODE_END(efi_enter_kernel) diff --git a/arch/arm64/kernel/efi-rt-wrapper.S b/arch/arm64/kernel/efi-rt-wrapper.S index 75691a2641c1..a00886410537 100644 --- a/arch/arm64/kernel/efi-rt-wrapper.S +++ b/arch/arm64/kernel/efi-rt-wrapper.S @@ -6,7 +6,7 @@ #include <linux/linkage.h> SYM_FUNC_START(__efi_rt_asm_wrapper) - stp x29, x30, [sp, #-32]! + stp x29, x30, [sp, #-112]! mov x29, sp /* @@ -17,6 +17,22 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) stp x1, x18, [sp, #16] /* + * Preserve all callee saved registers and preserve the stack pointer + * value at the base of the EFI runtime stack so we can recover from + * synchronous exceptions occurring while executing the firmware + * routines. + */ + stp x19, x20, [sp, #32] + stp x21, x22, [sp, #48] + stp x23, x24, [sp, #64] + stp x25, x26, [sp, #80] + stp x27, x28, [sp, #96] + + ldr_l x16, efi_rt_stack_top + mov sp, x16 + stp x18, x29, [sp, #-16]! + + /* * We are lucky enough that no EFI runtime services take more than * 5 arguments, so all are passed in registers rather than via the * stack. @@ -29,9 +45,10 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) mov x4, x6 blr x8 + mov sp, x29 ldp x1, x2, [sp, #16] cmp x2, x18 - ldp x29, x30, [sp], #32 + ldp x29, x30, [sp], #112 b.ne 0f ret 0: @@ -42,6 +59,22 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) * called with preemption disabled and a separate shadow stack is used * for interrupts. */ - mov x18, x2 +#ifdef CONFIG_SHADOW_CALL_STACK + ldr_l x18, efi_rt_stack_top + ldr x18, [x18, #-16] +#endif + b efi_handle_corrupted_x18 // tail call SYM_FUNC_END(__efi_rt_asm_wrapper) + +SYM_CODE_START(__efi_rt_asm_recover) + mov sp, x30 + + ldp x19, x20, [sp, #32] + ldp x21, x22, [sp, #48] + ldp x23, x24, [sp, #64] + ldp x25, x26, [sp, #80] + ldp x27, x28, [sp, #96] + ldp x29, x30, [sp], #112 + ret +SYM_CODE_END(__efi_rt_asm_recover) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index a908a37f0367..fab05de2e12d 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -144,3 +144,52 @@ asmlinkage efi_status_t efi_handle_corrupted_x18(efi_status_t s, const char *f) pr_err_ratelimited(FW_BUG "register x18 corrupted by EFI %s\n", f); return s; } + +DEFINE_SPINLOCK(efi_rt_lock); + +asmlinkage u64 *efi_rt_stack_top __ro_after_init; + +asmlinkage efi_status_t __efi_rt_asm_recover(void); + +bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg) +{ + /* Check whether the exception occurred while running the firmware */ + if (current_work() != &efi_rts_work.work || regs->pc >= TASK_SIZE_64) + return false; + + pr_err(FW_BUG "Unable to handle %s in EFI runtime service\n", msg); + add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); + clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); + + regs->regs[0] = EFI_ABORTED; + regs->regs[30] = efi_rt_stack_top[-1]; + regs->pc = (u64)__efi_rt_asm_recover; + + if (IS_ENABLED(CONFIG_SHADOW_CALL_STACK)) + regs->regs[18] = efi_rt_stack_top[-2]; + + return true; +} + +/* EFI requires 8 KiB of stack space for runtime services */ +static_assert(THREAD_SIZE >= SZ_8K); + +static int __init arm64_efi_rt_init(void) +{ + void *p; + + if (!efi_enabled(EFI_RUNTIME_SERVICES)) + return 0; + + p = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, GFP_KERNEL, + NUMA_NO_NODE, &&l); +l: if (!p) { + pr_warn("Failed to allocate EFI runtime stack\n"); + clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); + return -ENOMEM; + } + + efi_rt_stack_top = p + THREAD_SIZE; + return 0; +} +core_initcall(arm64_efi_rt_init); diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index 8151412653de..f31130ba0233 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -10,7 +10,6 @@ #error This file should only be included in vmlinux.lds.S #endif -PROVIDE(__efistub_kernel_size = _edata - _text); PROVIDE(__efistub_primary_entry_offset = primary_entry - _text); /* @@ -22,13 +21,6 @@ PROVIDE(__efistub_primary_entry_offset = primary_entry - _text); * linked at. The routines below are all implemented in assembler in a * position independent manner */ -PROVIDE(__efistub_memcmp = __pi_memcmp); -PROVIDE(__efistub_memchr = __pi_memchr); -PROVIDE(__efistub_strlen = __pi_strlen); -PROVIDE(__efistub_strnlen = __pi_strnlen); -PROVIDE(__efistub_strcmp = __pi_strcmp); -PROVIDE(__efistub_strncmp = __pi_strncmp); -PROVIDE(__efistub_strrchr = __pi_strrchr); PROVIDE(__efistub_dcache_clean_poc = __pi_dcache_clean_poc); PROVIDE(__efistub__text = _text); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 74f76514a48d..3eb2825d08cf 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -30,6 +30,7 @@ #include <asm/bug.h> #include <asm/cmpxchg.h> #include <asm/cpufeature.h> +#include <asm/efi.h> #include <asm/exception.h> #include <asm/daifflags.h> #include <asm/debug-monitors.h> @@ -397,6 +398,9 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr, msg = "paging request"; } + if (efi_runtime_fixup_exception(regs, msg)) + return; + die_kernel_fault(msg, addr, esr, regs); } diff --git a/arch/loongarch/include/asm/efi.h b/arch/loongarch/include/asm/efi.h index 174567b00ddb..97f16e60c6ff 100644 --- a/arch/loongarch/include/asm/efi.h +++ b/arch/loongarch/include/asm/efi.h @@ -19,18 +19,18 @@ void efifb_setup_from_dmi(struct screen_info *si, const char *opt); #define EFI_ALLOC_ALIGN SZ_64K #define EFI_RT_VIRTUAL_OFFSET CSR_DMW0_BASE -static inline struct screen_info *alloc_screen_info(void) +static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr) { - return &screen_info; + return ULONG_MAX; } -static inline void free_screen_info(struct screen_info *si) +static inline unsigned long efi_get_kimg_min_align(void) { + return SZ_2M; } -static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr) -{ - return ULONG_MAX; -} +#define EFI_KIMG_PREFERRED_ADDRESS PHYSADDR(VMLINUX_LOAD_ADDRESS) + +unsigned long kernel_entry_address(void); #endif /* _ASM_LOONGARCH_EFI_H */ diff --git a/arch/loongarch/kernel/efi.c b/arch/loongarch/kernel/efi.c index a31329971133..d75ce73e8ff8 100644 --- a/arch/loongarch/kernel/efi.c +++ b/arch/loongarch/kernel/efi.c @@ -52,6 +52,27 @@ void __init efi_runtime_init(void) set_bit(EFI_RUNTIME_SERVICES, &efi.flags); } +unsigned long __initdata screen_info_table = EFI_INVALID_TABLE_ADDR; + +static void __init init_screen_info(void) +{ + struct screen_info *si; + + if (screen_info_table == EFI_INVALID_TABLE_ADDR) + return; + + si = early_memremap(screen_info_table, sizeof(*si)); + if (!si) { + pr_err("Could not map screen_info config table\n"); + return; + } + screen_info = *si; + memset(si, 0, sizeof(*si)); + early_memunmap(si, sizeof(*si)); + + memblock_reserve(screen_info.lfb_base, screen_info.lfb_size); +} + void __init efi_init(void) { int size; @@ -80,8 +101,7 @@ void __init efi_init(void) set_bit(EFI_CONFIG_TABLES, &efi.flags); - if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI) - memblock_reserve(screen_info.lfb_base, screen_info.lfb_size); + init_screen_info(); if (boot_memmap == EFI_INVALID_TABLE_ADDR) return; diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 84970e266658..57bada6b4e93 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -25,7 +25,8 @@ _head: .dword kernel_entry /* Kernel entry point */ .dword _end - _text /* Kernel image effective size */ .quad 0 /* Kernel image load offset from start of RAM */ - .org 0x3c /* 0x20 ~ 0x3b reserved */ + .org 0x38 /* 0x20 ~ 0x37 reserved */ + .long LINUX_PE_MAGIC .long pe_header - _head /* Offset to the PE header */ pe_header: diff --git a/arch/loongarch/kernel/image-vars.h b/arch/loongarch/kernel/image-vars.h index 88f5d81702df..e561989d02de 100644 --- a/arch/loongarch/kernel/image-vars.h +++ b/arch/loongarch/kernel/image-vars.h @@ -7,15 +7,7 @@ #ifdef CONFIG_EFI_STUB -__efistub_memcmp = memcmp; -__efistub_memchr = memchr; -__efistub_strcat = strcat; __efistub_strcmp = strcmp; -__efistub_strlen = strlen; -__efistub_strncat = strncat; -__efistub_strnstr = strnstr; -__efistub_strnlen = strnlen; -__efistub_strrchr = strrchr; __efistub_kernel_entry = kernel_entry; __efistub_kernel_asize = kernel_asize; __efistub_kernel_fsize = kernel_fsize; diff --git a/arch/riscv/include/asm/efi.h b/arch/riscv/include/asm/efi.h index e229d7be4b66..47d3ab0fcc36 100644 --- a/arch/riscv/include/asm/efi.h +++ b/arch/riscv/include/asm/efi.h @@ -35,13 +35,20 @@ static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr) |