diff options
50 files changed, 430 insertions, 287 deletions
diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index c392e18f1e43..9afdc4c4a5dc 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -164,9 +164,6 @@ static int xen_starting_cpu(unsigned int cpu) BUG_ON(err); per_cpu(xen_vcpu, cpu) = vcpup; - if (!xen_kernel_unmapped_at_usr()) - xen_setup_runstate_info(cpu); - after_register_vcpu_info: enable_percpu_irq(xen_events_irq, 0); return 0; @@ -523,9 +520,6 @@ static int __init xen_guest_init(void) return -EINVAL; } - if (!xen_kernel_unmapped_at_usr()) - xen_time_setup_guest(); - if (xen_initial_domain()) pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier); @@ -535,7 +529,13 @@ static int __init xen_guest_init(void) } early_initcall(xen_guest_init); -static int __init xen_pm_init(void) +static int xen_starting_runstate_cpu(unsigned int cpu) +{ + xen_setup_runstate_info(cpu); + return 0; +} + +static int __init xen_late_init(void) { if (!xen_domain()) return -ENODEV; @@ -548,9 +548,16 @@ static int __init xen_pm_init(void) do_settimeofday64(&ts); } - return 0; + if (xen_kernel_unmapped_at_usr()) + return 0; + + xen_time_setup_guest(); + + return cpuhp_setup_state(CPUHP_AP_ARM_XEN_RUNSTATE_STARTING, + "arm/xen_runstate:starting", + xen_starting_runstate_cpu, NULL); } -late_initcall(xen_pm_init); +late_initcall(xen_late_init); /* empty stubs */ diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index 5c8ee5a541d2..4b6d2d52053e 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -6,5 +6,5 @@ generic-y += qspinlock.h generic-y += parport.h generic-y += user.h -generated-y += cpucaps.h +generated-y += cpucap-defs.h generated-y += sysreg-defs.h diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h index 94b486192e1f..210bb43cff2c 100644 --- a/arch/arm64/include/asm/alternative-macros.h +++ b/arch/arm64/include/asm/alternative-macros.h @@ -226,8 +226,8 @@ alternative_endif static __always_inline bool alternative_has_cap_likely(const unsigned long cpucap) { - compiletime_assert(cpucap < ARM64_NCAPS, - "cpucap must be < ARM64_NCAPS"); + if (!cpucap_is_possible(cpucap)) + return false; asm_volatile_goto( ALTERNATIVE_CB("b %l[l_no]", %[cpucap], alt_cb_patch_nops) @@ -244,8 +244,8 @@ l_no: static __always_inline bool alternative_has_cap_unlikely(const unsigned long cpucap) { - compiletime_assert(cpucap < ARM64_NCAPS, - "cpucap must be < ARM64_NCAPS"); + if (!cpucap_is_possible(cpucap)) + return false; asm_volatile_goto( ALTERNATIVE("nop", "b %l[l_yes]", %[cpucap]) diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index 01281a5336cf..5f172611654b 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -79,6 +79,14 @@ static inline u64 gic_read_iar_cavium_thunderx(void) return 0x3ff; } +static u64 __maybe_unused gic_read_iar(void) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_CAVIUM_23154)) + return gic_read_iar_cavium_thunderx(); + else + return gic_read_iar_common(); +} + static inline void gic_write_ctlr(u32 val) { write_sysreg_s(val, SYS_ICC_CTLR_EL1); diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h index b0abc64f86b0..ecdb3cfcd0f8 100644 --- a/arch/arm64/include/asm/archrandom.h +++ b/arch/arm64/include/asm/archrandom.h @@ -63,7 +63,7 @@ static __always_inline bool __cpu_has_rng(void) { if (unlikely(!system_capabilities_finalized() && !preemptible())) return this_cpu_has_cap(ARM64_HAS_RNG); - return cpus_have_const_cap(ARM64_HAS_RNG); + return alternative_has_cap_unlikely(ARM64_HAS_RNG); } static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs) diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index d115451ed263..fefac75fa009 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -132,7 +132,7 @@ void flush_dcache_folio(struct folio *); static __always_inline void icache_inval_all_pou(void) { - if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC)) + if (alternative_has_cap_unlikely(ARM64_HAS_CACHE_DIC)) return; asm("ic ialluis"); diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h new file mode 100644 index 000000000000..270680e2b5c4 --- /dev/null +++ b/arch/arm64/include/asm/cpucaps.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_CPUCAPS_H +#define __ASM_CPUCAPS_H + +#include <asm/cpucap-defs.h> + +#ifndef __ASSEMBLY__ +#include <linux/types.h> +/* + * Check whether a cpucap is possible at compiletime. + */ +static __always_inline bool +cpucap_is_possible(const unsigned int cap) +{ + compiletime_assert(__builtin_constant_p(cap), + "cap must be a constant"); + compiletime_assert(cap < ARM64_NCAPS, + "cap must be < ARM64_NCAPS"); + + switch (cap) { + case ARM64_HAS_PAN: + return IS_ENABLED(CONFIG_ARM64_PAN); + case ARM64_HAS_EPAN: + return IS_ENABLED(CONFIG_ARM64_EPAN); + case ARM64_SVE: + return IS_ENABLED(CONFIG_ARM64_SVE); + case ARM64_SME: + case ARM64_SME2: + case ARM64_SME_FA64: + return IS_ENABLED(CONFIG_ARM64_SME); + case ARM64_HAS_CNP: + return IS_ENABLED(CONFIG_ARM64_CNP); + case ARM64_HAS_ADDRESS_AUTH: + case ARM64_HAS_GENERIC_AUTH: + return IS_ENABLED(CONFIG_ARM64_PTR_AUTH); + case ARM64_HAS_GIC_PRIO_MASKING: + return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI); + case ARM64_MTE: + return IS_ENABLED(CONFIG_ARM64_MTE); + case ARM64_BTI: + return IS_ENABLED(CONFIG_ARM64_BTI); + case ARM64_HAS_TLB_RANGE: + return IS_ENABLED(CONFIG_ARM64_TLB_RANGE); + case ARM64_UNMAP_KERNEL_AT_EL0: + return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0); + case ARM64_WORKAROUND_843419: + return IS_ENABLED(CONFIG_ARM64_ERRATUM_843419); + case ARM64_WORKAROUND_1742098: + return IS_ENABLED(CONFIG_ARM64_ERRATUM_1742098); + case ARM64_WORKAROUND_2645198: + return IS_ENABLED(CONFIG_ARM64_ERRATUM_2645198); + case ARM64_WORKAROUND_2658417: + return IS_ENABLED(CONFIG_ARM64_ERRATUM_2658417); + case ARM64_WORKAROUND_CAVIUM_23154: + return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154); + case ARM64_WORKAROUND_NVIDIA_CARMEL_CNP: + return IS_ENABLED(CONFIG_NVIDIA_CARMEL_CNP_ERRATUM); + case ARM64_WORKAROUND_REPEAT_TLBI: + return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); + } + + return true; +} +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_CPUCAPS_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 19b4d001d845..f6d416fe49b0 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -440,6 +440,11 @@ unsigned long cpu_get_elf_hwcap2(void); #define cpu_set_named_feature(name) cpu_set_feature(cpu_feature(name)) #define cpu_have_named_feature(name) cpu_have_feature(cpu_feature(name)) +static __always_inline bool boot_capabilities_finalized(void) +{ + return alternative_has_cap_likely(ARM64_ALWAYS_BOOT); +} + static __always_inline bool system_capabilities_finalized(void) { return alternative_has_cap_likely(ARM64_ALWAYS_SYSTEM); @@ -452,6 +457,8 @@ static __always_inline bool system_capabilities_finalized(void) */ static __always_inline bool cpus_have_cap(unsigned int num) { + if (__builtin_constant_p(num) && !cpucap_is_possible(num)) + return false; if (num >= ARM64_NCAPS) return false; return arch_test_bit(num, system_cpucaps); @@ -460,55 +467,37 @@ static __always_inline bool cpus_have_cap(unsigned int num) /* * Test for a capability without a runtime check. * - * Before capabilities are finalized, this returns false. - * After capabilities are finalized, this is patched to avoid a runtime check. + * Before boot capabilities are finalized, this will BUG(). + * After boot capabilities are finalized, this is patched to avoid a runtime + * check. * * @num must be a compile-time constant. */ -static __always_inline bool __cpus_have_const_cap(int num) +static __always_inline bool cpus_have_final_boot_cap(int num) { - if (num >= ARM64_NCAPS) - return false; - return alternative_has_cap_unlikely(num); + if (boot_capabilities_finalized()) + return alternative_has_cap_unlikely(num); + else + BUG(); } /* * Test for a capability without a runtime check. * - * Before capabilities are finalized, this will BUG(). - * After capabilities are finalized, this is patched to avoid a runtime check. + * Before system capabilities are finalized, this will BUG(). + * After system capabilities are finalized, this is patched to avoid a runtime + * check. * * @num must be a compile-time constant. */ static __always_inline bool cpus_have_final_cap(int num) { if (system_capabilities_finalized()) - return __cpus_have_const_cap(num); + return alternative_has_cap_unlikely(num); else BUG(); } -/* - * Test for a capability, possibly with a runtime check for non-hyp code. - * - * For hyp code, this behaves the same as cpus_have_final_cap(). - * - * For non-hyp code: - * Before capabilities are finalized, this behaves as cpus_have_cap(). - * After capabilities are finalized, this is patched to avoid a runtime check. - * - * @num must be a compile-time constant. - */ -static __always_inline bool cpus_have_const_cap(int num) -{ - if (is_hyp_code()) - return cpus_have_final_cap(num); - else if (system_capabilities_finalized()) - return __cpus_have_const_cap(num); - else - return cpus_have_cap(num); -} - static inline int __attribute_const__ cpuid_feature_extract_signed_field_width(u64 features, int field, int width) { @@ -628,7 +617,9 @@ static inline bool id_aa64pfr1_mte(u64 pfr1) return val >= ID_AA64PFR1_EL1_MTE_MTE2; } -void __init setup_cpu_features(void); +void __init setup_system_features(void); +void __init setup_user_features(void); + void check_local_cpu_capabilities(void); u64 read_sanitised_ftr_reg(u32 id); @@ -737,13 +728,12 @@ static inline bool system_supports_mixed_endian(void) static __always_inline bool system_supports_fpsimd(void) { - return !cpus_have_const_cap(ARM64_HAS_NO_FPSIMD); + return alternative_has_cap_likely(ARM64_HAS_FPSIMD); } static inline bool system_uses_hw_pan(void) { - return IS_ENABLED(CONFIG_ARM64_PAN) && - cpus_have_const_cap(ARM64_HAS_PAN); + return alternative_has_cap_unlikely(ARM64_HAS_PAN); } static inline bool system_uses_ttbr0_pan(void) @@ -754,26 +744,22 @@ static inline bool system_uses_ttbr0_pan(void) static __always_inline bool system_supports_sve(void) { - return IS_ENABLED(CONFIG_ARM64_SVE) && - cpus_have_const_cap(ARM64_SVE); + return alternative_has_cap_unlikely(ARM64_SVE); } static __always_inline bool system_supports_sme(void) { - return IS_ENABLED(CONFIG_ARM64_SME) && - cpus_have_const_cap(ARM64_SME); + return alternative_has_cap_unlikely(ARM64_SME); } static __always_inline bool system_supports_sme2(void) { - return IS_ENABLED(CONFIG_ARM64_SME) && - cpus_have_const_cap(ARM64_SME2); + return alternative_has_cap_unlikely(ARM64_SME2); } static __always_inline bool system_supports_fa64(void) { - return IS_ENABLED(CONFIG_ARM64_SME) && - cpus_have_const_cap(ARM64_SME_FA64); + return alternative_has_cap_unlikely(ARM64_SME_FA64); } static __always_inline bool system_supports_tpidr2(void) @@ -783,20 +769,17 @@ static __always_inline bool system_supports_tpidr2(void) static __always_inline bool system_supports_cnp(void) { - return IS_ENABLED(CONFIG_ARM64_CNP) && - cpus_have_const_cap(ARM64_HAS_CNP); + return alternative_has_cap_unlikely(ARM64_HAS_CNP); } static inline bool system_supports_address_auth(void) { - return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) && - cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH); + return cpus_have_final_boot_cap(ARM64_HAS_ADDRESS_AUTH); } static inline bool system_supports_generic_auth(void) { - return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) && - cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH); + return alternative_has_cap_unlikely(ARM64_HAS_GENERIC_AUTH); } static inline bool system_has_full_ptr_auth(void) @@ -806,14 +789,12 @@ static inline bool system_has_full_ptr_auth(void) static __always_inline bool system_uses_irq_prio_masking(void) { - return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && - cpus_have_const_cap(ARM64_HAS_GIC_PRIO_MASKING); + return alternative_has_cap_unlikely(ARM64_HAS_GIC_PRIO_MASKING); } static inline bool system_supports_mte(void) { - return IS_ENABLED(CONFIG_ARM64_MTE) && - cpus_have_const_cap(ARM64_MTE); + return alternative_has_cap_unlikely(ARM64_MTE); } static inline bool system_has_prio_mask_debugging(void) @@ -824,13 +805,18 @@ static inline bool system_has_prio_mask_debugging(void) static inline bool system_supports_bti(void) { - return IS_ENABLED(CONFIG_ARM64_BTI) && cpus_have_const_cap(ARM64_BTI); + return cpus_have_final_cap(ARM64_BTI); +} + +static inline bool system_supports_bti_kernel(void) +{ + return IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && + cpus_have_final_boot_cap(ARM64_BTI); } static inline bool system_supports_tlb_range(void) { - return IS_ENABLED(CONFIG_ARM64_TLB_RANGE) && - cpus_have_const_cap(ARM64_HAS_TLB_RANGE); + return alternative_has_cap_unlikely(ARM64_HAS_TLB_RANGE); } int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 9e5d3a0812b6..50e5f25d3024 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -32,6 +32,32 @@ #define VFP_STATE_SIZE ((32 * 8) + 4) #endif +static inline unsigned long cpacr_save_enable_kernel_sve(void) +{ + unsigned long old = read_sysreg(cpacr_el1); + unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_ZEN_EL1EN; + + write_sysreg(old | set, cpacr_el1); + isb(); + return old; +} + +static inline unsigned long cpacr_save_enable_kernel_sme(void) +{ + unsigned long old = read_sysreg(cpacr_el1); + unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_SMEN_EL1EN; + + write_sysreg(old | set, cpacr_el1); + isb(); + return old; +} + +static inline void cpacr_restore(unsigned long cpacr) +{ + write_sysreg(cpacr, cpacr_el1); + isb(); +} + /* * When we defined the maximum SVE vector length we defined the ABI so * that the maximum vector length included all the reserved for future @@ -123,10 +149,11 @@ extern void sme_save_state(void *state, int zt); extern void sme_load_state(void const *state, int zt); struct arm64_cpu_capabilities; -extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused); -extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused); -extern void sme2_kernel_enable(const struct arm64_cpu_capabilities *__unused); -extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused); +extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused); +extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused); +extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused); +extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused); +extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused); extern u64 read_smcr_features(void); diff --git a/arch/arm64/include/asm/irqflags.h b/arch/arm64/include/asm/irqflags.h index 1f31ec146d16..0a7186a93882 100644 --- a/arch/arm64/include/asm/irqflags.h +++ b/arch/arm64/include/asm/irqflags.h @@ -21,12 +21,6 @@ * exceptions should be unmasked. */ -static __always_inline bool __irqflags_uses_pmr(void) -{ - return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && - alternative_has_cap_unlikely(ARM64_HAS_GIC_PRIO_MASKING); -} - static __always_inline void __daif_local_irq_enable(void) { barrier(); @@ -49,7 +43,7 @@ static __always_inline void __pmr_local_irq_enable(void) static inline void arch_local_irq_enable(void) { - if (__irqflags_uses_pmr()) { + if (system_uses_irq_prio_masking()) { __pmr_local_irq_enable(); } else { __daif_local_irq_enable(); @@ -77,7 +71,7 @@ static __always_inline void __pmr_local_irq_disable(void) static inline void arch_local_irq_disable(void) { - if (__irqflags_uses_pmr()) { + if (system_uses_irq_prio_masking()) { __pmr_local_irq_disable(); } else { __daif_local_irq_disable(); @@ -99,7 +93,7 @@ static __always_inline unsigned long __pmr_local_save_flags(void) */ static inline unsigned long arch_local_save_flags(void) { - if (__irqflags_uses_pmr()) { + if (system_uses_irq_prio_masking()) { return __pmr_local_save_flags(); } else { return __daif_local_save_flags(); @@ -118,7 +112,7 @@ static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags) static inline bool arch_irqs_disabled_flags(unsigned long flags) { - if (__irqflags_uses_pmr()) { + if (system_uses_irq_prio_masking()) { return __pmr_irqs_disabled_flags(flags); } else { return __daif_irqs_disabled_flags(flags); @@ -137,7 +131,7 @@ static __always_inline bool __pmr_irqs_disabled(void) static inline bool arch_irqs_disabled(void) { - if (__irqflags_uses_pmr()) { + if (system_uses_irq_prio_masking()) { return __pmr_irqs_disabled(); } else { return __daif_irqs_disabled(); @@ -169,7 +163,7 @@ static __always_inline unsigned long __pmr_local_irq_save(void) static inline unsigned long arch_local_irq_save(void) { - if (__irqflags_uses_pmr()) { + if (system_uses_irq_prio_masking()) { return __pmr_local_irq_save(); } else { return __daif_local_irq_save(); @@ -196,7 +190,7 @@ static __always_inline void __pmr_local_irq_restore(unsigned long flags) */ static inline void arch_local_irq_restore(unsigned long flags) { - if (__irqflags_uses_pmr()) { + if (system_uses_irq_prio_masking()) { __pmr_local_irq_restore(flags); } else { __daif_local_irq_restore(flags); diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 3d6725ff0bf6..cbd2f163a67d 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -71,14 +71,14 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; if (has_vhe() || has_hvhe()) vcpu->arch.hcr_el2 |= HCR_E2H; - if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) { + if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) { /* route synchronous external abort exceptions to EL2 */ vcpu->arch.hcr_el2 |= HCR_TEA; /* trap error record accesses */ vcpu->arch.hcr_el2 |= HCR_TERR; } - if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) { + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) { vcpu->arch.hcr_el2 |= HCR_FWB; } else { /* diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index af06ccb7ee34..e64d64e6ad44 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1052,7 +1052,7 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt) static inline bool kvm_system_needs_idmapped_vectors(void) { - return cpus_have_const_cap(ARM64_SPECTRE_V3A); + return cpus_have_final_cap(ARM64_SPECTRE_V3A); } static inline void kvm_arch_sync_events(struct kvm *kvm) {} diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 96a80e8f6226..27810667dec7 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -218,7 +218,7 @@ static inline void __clean_dcache_guest_page(void *va, size_t size) * faulting in pages. Furthermore, FWB implies IDC, so cleaning to * PoU is not required either in this case. */ - if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) return; kvm_flush_dcache_to_poc(va, size); diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 94b68850cb9f..2fcf51231d6e 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -57,7 +57,7 @@ typedef struct { static inline bool arm64_kernel_unmapped_at_el0(void) { - return cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0); + return alternative_has_cap_unlikely(ARM64_UNMAP_KERNEL_AT_EL0); } extern void arm64_memblock_init(void); diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index a6fb325424e7..9ce4200508b1 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -152,7 +152,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD, * avoiding the possibility of conflicting TLB entries being allocated. */ -static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) +static inline void __cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap, bool cnp) { typedef void (ttbr_replace_func)(phys_addr_t); extern ttbr_replace_func idmap_cpu_replace_ttbr1; @@ -162,17 +162,8 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) /* phys_to_ttbr() zeros lower 2 bits of ttbr with 52-bit PA */ phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp)); - if (system_supports_cnp() && !WARN_ON(pgdp != lm_alias(swapper_pg_dir))) { - /* |