diff options
72 files changed, 2391 insertions, 995 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 1394a8491210..749d6e3788b4 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -35,6 +35,7 @@ config ARM select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7 select ARCH_SUPPORTS_ATOMIC_RMW + select ARCH_SUPPORTS_CFI_CLANG select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE select ARCH_SUPPORTS_PER_VMA_LOCK select ARCH_USE_BUILTIN_BSWAP diff --git a/arch/arm/include/asm/glue-cache.h b/arch/arm/include/asm/glue-cache.h index 724f8dac1e5b..4186fbf7341f 100644 --- a/arch/arm/include/asm/glue-cache.h +++ b/arch/arm/include/asm/glue-cache.h @@ -118,6 +118,10 @@ # define MULTI_CACHE 1 #endif +#ifdef CONFIG_CPU_CACHE_NOP +# define MULTI_CACHE 1 +#endif + #if defined(CONFIG_CPU_V7M) # define MULTI_CACHE 1 #endif @@ -126,29 +130,15 @@ #error Unknown cache maintenance model #endif -#ifndef __ASSEMBLER__ -static inline void nop_flush_icache_all(void) { } -static inline void nop_flush_kern_cache_all(void) { } -static inline void nop_flush_kern_cache_louis(void) { } -static inline void nop_flush_user_cache_all(void) { } -static inline void nop_flush_user_cache_range(unsigned long a, - unsigned long b, unsigned int c) { } - -static inline void nop_coherent_kern_range(unsigned long a, unsigned long b) { } -static inline int nop_coherent_user_range(unsigned long a, - unsigned long b) { return 0; } -static inline void nop_flush_kern_dcache_area(void *a, size_t s) { } - -static inline void nop_dma_flush_range(const void *a, const void *b) { } - -static inline void nop_dma_map_area(const void *s, size_t l, int f) { } -static inline void nop_dma_unmap_area(const void *s, size_t l, int f) { } -#endif - #ifndef MULTI_CACHE #define __cpuc_flush_icache_all __glue(_CACHE,_flush_icache_all) #define __cpuc_flush_kern_all __glue(_CACHE,_flush_kern_cache_all) +/* This function only has a dedicated assembly callback on the v7 cache */ +#ifdef CONFIG_CPU_CACHE_V7 #define __cpuc_flush_kern_louis __glue(_CACHE,_flush_kern_cache_louis) +#else +#define __cpuc_flush_kern_louis __glue(_CACHE,_flush_kern_cache_all) +#endif #define __cpuc_flush_user_all __glue(_CACHE,_flush_user_cache_all) #define __cpuc_flush_user_range __glue(_CACHE,_flush_user_cache_range) #define __cpuc_coherent_kern_range __glue(_CACHE,_coherent_kern_range) diff --git a/arch/arm/include/asm/hw_breakpoint.h b/arch/arm/include/asm/hw_breakpoint.h index 62358d3ca0a8..e7f9961c53b2 100644 --- a/arch/arm/include/asm/hw_breakpoint.h +++ b/arch/arm/include/asm/hw_breakpoint.h @@ -84,6 +84,7 @@ static inline void decode_ctrl_reg(u32 reg, #define ARM_DSCR_MOE(x) ((x >> 2) & 0xf) #define ARM_ENTRY_BREAKPOINT 0x1 #define ARM_ENTRY_ASYNC_WATCHPOINT 0x2 +#define ARM_ENTRY_CFI_BREAKPOINT 0x3 #define ARM_ENTRY_SYNC_WATCHPOINT 0xa /* DSCR monitor/halting bits. */ diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index dc0fb7a81371..93f9034e4e72 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -17,6 +17,7 @@ #include <linux/perf_event.h> #include <linux/hw_breakpoint.h> #include <linux/smp.h> +#include <linux/cfi.h> #include <linux/cpu_pm.h> #include <linux/coresight.h> @@ -903,6 +904,37 @@ unlock: watchpoint_single_step_handler(addr); } +#ifdef CONFIG_CFI_CLANG +static void hw_breakpoint_cfi_handler(struct pt_regs *regs) +{ + /* + * TODO: implementing target and type to pass to CFI using the more + * elaborate report_cfi_failure() requires compiler work. To be able + * to properly extract target information the compiler needs to + * emit a stable instructions sequence for the CFI checks so we can + * decode the instructions preceding the trap and figure out which + * registers were used. + */ + + switch (report_cfi_failure_noaddr(regs, instruction_pointer(regs))) { + case BUG_TRAP_TYPE_BUG: + die("Oops - CFI", regs, 0); + break; + case BUG_TRAP_TYPE_WARN: + /* Skip the breaking instruction */ + instruction_pointer(regs) += 4; + break; + default: + die("Unknown CFI error", regs, 0); + break; + } +} +#else +static void hw_breakpoint_cfi_handler(struct pt_regs *regs) +{ +} +#endif + /* * Called from either the Data Abort Handler [watchpoint] or the * Prefetch Abort Handler [breakpoint] with interrupts disabled. @@ -932,6 +964,9 @@ static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, case ARM_ENTRY_SYNC_WATCHPOINT: watchpoint_handler(addr, fsr, regs); break; + case ARM_ENTRY_CFI_BREAKPOINT: + hw_breakpoint_cfi_handler(regs); + break; default: ret = 1; /* Unhandled fault. */ } diff --git a/arch/arm/lib/delay-loop.S b/arch/arm/lib/delay-loop.S index 3ac05177d097..33b08ca1c242 100644 --- a/arch/arm/lib/delay-loop.S +++ b/arch/arm/lib/delay-loop.S @@ -5,6 +5,7 @@ * Copyright (C) 1995, 1996 Russell King */ #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/delay.h> @@ -24,21 +25,26 @@ * HZ <= 1000 */ -ENTRY(__loop_udelay) +SYM_TYPED_FUNC_START(__loop_udelay) ldr r2, .LC1 mul r0, r2, r0 @ r0 = delay_us * UDELAY_MULT -ENTRY(__loop_const_udelay) @ 0 <= r0 <= 0xfffffaf0 + b __loop_const_udelay +SYM_FUNC_END(__loop_udelay) + +SYM_TYPED_FUNC_START(__loop_const_udelay) @ 0 <= r0 <= 0xfffffaf0 ldr r2, .LC0 ldr r2, [r2] umull r1, r0, r2, r0 @ r0-r1 = r0 * loops_per_jiffy adds r1, r1, #0xffffffff @ rounding up ... adcs r0, r0, r0 @ and right shift by 31 reteq lr + b __loop_delay +SYM_FUNC_END(__loop_const_udelay) .align 3 @ Delay routine -ENTRY(__loop_delay) +SYM_TYPED_FUNC_START(__loop_delay) subs r0, r0, #1 #if 0 retls lr @@ -58,6 +64,4 @@ ENTRY(__loop_delay) #endif bhi __loop_delay ret lr -ENDPROC(__loop_udelay) -ENDPROC(__loop_const_udelay) -ENDPROC(__loop_delay) +SYM_FUNC_END(__loop_delay) diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 71b858c9b10c..f1f231f20ff9 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -45,6 +45,7 @@ obj-$(CONFIG_CPU_CACHE_V7) += cache-v7.o obj-$(CONFIG_CPU_CACHE_FA) += cache-fa.o obj-$(CONFIG_CPU_CACHE_NOP) += cache-nop.o obj-$(CONFIG_CPU_CACHE_V7M) += cache-v7m.o +obj-y += cache.o obj-$(CONFIG_CPU_COPY_V4WT) += copypage-v4wt.o obj-$(CONFIG_CPU_COPY_V4WB) += copypage-v4wb.o @@ -62,6 +63,7 @@ obj-$(CONFIG_CPU_TLB_FEROCEON) += tlb-v4wbi.o # reuse v4wbi TLB functions obj-$(CONFIG_CPU_TLB_V6) += tlb-v6.o obj-$(CONFIG_CPU_TLB_V7) += tlb-v7.o obj-$(CONFIG_CPU_TLB_FA) += tlb-fa.o +obj-y += tlb.o obj-$(CONFIG_CPU_ARM7TDMI) += proc-arm7tdmi.o obj-$(CONFIG_CPU_ARM720T) += proc-arm720.o @@ -88,6 +90,7 @@ obj-$(CONFIG_CPU_V6) += proc-v6.o obj-$(CONFIG_CPU_V6K) += proc-v6.o obj-$(CONFIG_CPU_V7) += proc-v7.o proc-v7-bugs.o obj-$(CONFIG_CPU_V7M) += proc-v7m.o +obj-$(CONFIG_CFI_CLANG) += proc.o obj-$(CONFIG_OUTER_CACHE) += l2c-common.o obj-$(CONFIG_CACHE_B15_RAC) += cache-b15-rac.o diff --git a/arch/arm/mm/cache-b15-rac.c b/arch/arm/mm/cache-b15-rac.c index 9c1172f26885..6f63b90f9e1a 100644 --- a/arch/arm/mm/cache-b15-rac.c +++ b/arch/arm/mm/cache-b15-rac.c @@ -5,6 +5,7 @@ * Copyright (C) 2015-2016 Broadcom */ +#include <linux/cfi_types.h> #include <linux/err.h> #include <linux/spinlock.h> #include <linux/io.h> diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S index 71c64e92dead..4a3668b52a2d 100644 --- a/arch/arm/mm/cache-fa.S +++ b/arch/arm/mm/cache-fa.S @@ -12,6 +12,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/page.h> @@ -39,11 +40,11 @@ * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(fa_flush_icache_all) +SYM_TYPED_FUNC_START(fa_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(fa_flush_icache_all) +SYM_FUNC_END(fa_flush_icache_all) /* * flush_user_cache_all() @@ -51,14 +52,14 @@ ENDPROC(fa_flush_icache_all) * Clean and invalidate all cache entries in a particular address * space. */ -ENTRY(fa_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(fa_flush_user_cache_all, fa_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(fa_flush_kern_cache_all) +SYM_TYPED_FUNC_START(fa_flush_kern_cache_all) mov ip, #0 mov r2, #VM_EXEC __flush_whole_cache: @@ -69,6 +70,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c10, 4 @ drain write buffer mcrne p15, 0, ip, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(fa_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -80,7 +82,7 @@ __flush_whole_cache: * - end - end address (exclusive, page aligned) * - flags - vma_area_struct flags describing address space */ -ENTRY(fa_flush_user_cache_range) +SYM_TYPED_FUNC_START(fa_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @ total size >= limit? @@ -97,6 +99,7 @@ ENTRY(fa_flush_user_cache_range) mcrne p15, 0, ip, c7, c10, 4 @ data write barrier mcrne p15, 0, ip, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(fa_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -108,8 +111,11 @@ ENTRY(fa_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(fa_coherent_kern_range) - /* fall through */ +SYM_TYPED_FUNC_START(fa_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b fa_coherent_user_range +#endif +SYM_FUNC_END(fa_coherent_kern_range) /* * coherent_user_range(start, end) @@ -121,7 +127,7 @@ ENTRY(fa_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(fa_coherent_user_range) +SYM_TYPED_FUNC_START(fa_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -133,6 +139,7 @@ ENTRY(fa_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain write buffer mcr p15, 0, r0, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(fa_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -143,7 +150,7 @@ ENTRY(fa_coherent_user_range) * - addr - kernel address * - size - size of region */ -ENTRY(fa_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(fa_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line add r0, r0, #CACHE_DLINESIZE @@ -153,6 +160,7 @@ ENTRY(fa_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain write buffer ret lr +SYM_FUNC_END(fa_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -203,7 +211,7 @@ fa_dma_clean_range: * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(fa_dma_flush_range) +SYM_TYPED_FUNC_START(fa_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -212,6 +220,7 @@ ENTRY(fa_dma_flush_range) mov r0, #0 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer ret lr +SYM_FUNC_END(fa_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -219,13 +228,13 @@ ENTRY(fa_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(fa_dma_map_area) +SYM_TYPED_FUNC_START(fa_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq fa_dma_clean_range bcs fa_dma_inv_range b fa_dma_flush_range -ENDPROC(fa_dma_map_area) +SYM_FUNC_END(fa_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -233,14 +242,6 @@ ENDPROC(fa_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(fa_dma_unmap_area) +SYM_TYPED_FUNC_START(fa_dma_unmap_area) ret lr -ENDPROC(fa_dma_unmap_area) - - .globl fa_flush_kern_cache_louis - .equ fa_flush_kern_cache_louis, fa_flush_kern_cache_all - - __INITDATA - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions fa +SYM_FUNC_END(fa_dma_unmap_area) diff --git a/arch/arm/mm/cache-nop.S b/arch/arm/mm/cache-nop.S index 72d939ef8798..f68dde2014ee 100644 --- a/arch/arm/mm/cache-nop.S +++ b/arch/arm/mm/cache-nop.S @@ -1,47 +1,52 @@ /* SPDX-License-Identifier: GPL-2.0-only */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include "proc-macros.S" -ENTRY(nop_flush_icache_all) +/* + * These are all open-coded instead of aliased, to make clear + * what is going on here: all functions are stubbed out. + */ +SYM_TYPED_FUNC_START(nop_flush_icache_all) ret lr -ENDPROC(nop_flush_icache_all) +SYM_FUNC_END(nop_flush_icache_all) - .globl nop_flush_kern_cache_all - .equ nop_flush_kern_cache_all, nop_flush_icache_all - - .globl nop_flush_kern_cache_louis - .equ nop_flush_kern_cache_louis, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_flush_kern_cache_all) + ret lr +SYM_FUNC_END(nop_flush_kern_cache_all) - .globl nop_flush_user_cache_all - .equ nop_flush_user_cache_all, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_flush_user_cache_all) + ret lr +SYM_FUNC_END(nop_flush_user_cache_all) - .globl nop_flush_user_cache_range - .equ nop_flush_user_cache_range, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_flush_user_cache_range) + ret lr +SYM_FUNC_END(nop_flush_user_cache_range) - .globl nop_coherent_kern_range - .equ nop_coherent_kern_range, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_coherent_kern_range) + ret lr +SYM_FUNC_END(nop_coherent_kern_range) -ENTRY(nop_coherent_user_range) +SYM_TYPED_FUNC_START(nop_coherent_user_range) mov r0, 0 ret lr -ENDPROC(nop_coherent_user_range) - - .globl nop_flush_kern_dcache_area - .equ nop_flush_kern_dcache_area, nop_flush_icache_all +SYM_FUNC_END(nop_coherent_user_range) - .globl nop_dma_flush_range - .equ nop_dma_flush_range, nop_flush_icache_all - - .globl nop_dma_map_area - .equ nop_dma_map_area, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_flush_kern_dcache_area) + ret lr +SYM_FUNC_END(nop_flush_kern_dcache_area) - .globl nop_dma_unmap_area - .equ nop_dma_unmap_area, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_dma_flush_range) |