x86/startup_64: Defer assignment of 5-level paging global variables

Assigning the 5-level paging related global variables from the earliest
C code using explicit references that use the 1:1 translation of memory
is unnecessary, as the startup code itself does not rely on them to
create the initial page tables, and this is all it should be doing. So
defer these assignments to the primary C entry code that executes via
the ordinary kernel virtual mapping.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Tested-by: Tom Lendacky <thomas.lendacky@amd.com>
Link: https://lore.kernel.org/r/20240227151907.387873-13-ardb+git@google.com

diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index 38b54b992f32e3027c5fa066f9684900fdd348d5..9053dfe9fa03f947f0ad28c20eb6bfd45856f7d3 100644 (file)
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -21,9 +21,9 @@ typedef unsigned long pgprotval_t;
 typedef struct { pteval_t pte; } pte_t;
 typedef struct { pmdval_t pmd; } pmd_t;
 
 typedef struct { pteval_t pte; } pte_t;
 typedef struct { pmdval_t pmd; } pmd_t;
 

-#ifdef CONFIG_X86_5LEVEL

 extern unsigned int __pgtable_l5_enabled;
 
 extern unsigned int __pgtable_l5_enabled;
 

+#ifdef CONFIG_X86_5LEVEL

 #ifdef USE_EARLY_PGTABLE_L5
 /*
  * cpu_feature_enabled() is not available in early boot code.
 #ifdef USE_EARLY_PGTABLE_L5
 /*
  * cpu_feature_enabled() is not available in early boot code.

diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index fd77a266f29dddce92ce0bbc03f47fadf076e1ee..212e8e06aeba23231a1dced48da0cdfe1c729c86 100644 (file)
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -23,6 +23,7 @@
 #include <linux/pgtable.h>
 
 #include <asm/asm.h>
 #include <linux/pgtable.h>
 
 #include <asm/asm.h>

+#include <asm/page_64.h>

 #include <asm/processor.h>
 #include <asm/proto.h>
 #include <asm/smp.h>
 #include <asm/processor.h>
 #include <asm/proto.h>
 #include <asm/smp.h>


@@ -68,24 +69,11 @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
 EXPORT_SYMBOL(vmemmap_base);
 #endif
 
 EXPORT_SYMBOL(vmemmap_base);
 #endif
 

-#ifdef CONFIG_X86_5LEVEL
-static void __head *fixup_pointer(void *ptr, unsigned long physaddr)
-{
-       return ptr - (void *)_text + (void *)physaddr;
-}
-
-static unsigned long __head *fixup_long(void *ptr, unsigned long physaddr)
+static inline bool check_la57_support(void)

 {
 {

-       return fixup_pointer(ptr, physaddr);
-}
-
-static unsigned int __head *fixup_int(void *ptr, unsigned long physaddr)
-{
-       return fixup_pointer(ptr, physaddr);
-}
+       if (!IS_ENABLED(CONFIG_X86_5LEVEL))
+               return false;

 
 

-static bool __head check_la57_support(unsigned long physaddr)
-{

        /*
         * 5-level paging is detected and enabled at kernel decompression
         * stage. Only check if it has been enabled there.
        /*
         * 5-level paging is detected and enabled at kernel decompression
         * stage. Only check if it has been enabled there.


@@ -93,21 +81,8 @@ static bool __head check_la57_support(unsigned long physaddr)
        if (!(native_read_cr4() & X86_CR4_LA57))
                return false;
 
        if (!(native_read_cr4() & X86_CR4_LA57))
                return false;
 

-       *fixup_int(&__pgtable_l5_enabled, physaddr) = 1;
-       *fixup_int(&pgdir_shift, physaddr) = 48;
-       *fixup_int(&ptrs_per_p4d, physaddr) = 512;
-       *fixup_long(&page_offset_base, physaddr) = __PAGE_OFFSET_BASE_L5;
-       *fixup_long(&vmalloc_base, physaddr) = __VMALLOC_BASE_L5;
-       *fixup_long(&vmemmap_base, physaddr) = __VMEMMAP_BASE_L5;
-

        return true;
 }
        return true;
 }

-#else
-static bool __head check_la57_support(unsigned long physaddr)
-{
-       return false;
-}
-#endif

 
 static unsigned long __head sme_postprocess_startup(struct boot_params *bp, pmdval_t *pmd)
 {
 
 static unsigned long __head sme_postprocess_startup(struct boot_params *bp, pmdval_t *pmd)
 {


@@ -171,7 +146,7 @@ unsigned long __head __startup_64(unsigned long physaddr,
        bool la57;
        int i;
 
        bool la57;
        int i;
 

-       la57 = check_la57_support(physaddr);
+       la57 = check_la57_support();

 
        /* Is the address too large? */
        if (physaddr >> MAX_PHYSMEM_BITS)
 
        /* Is the address too large? */
        if (physaddr >> MAX_PHYSMEM_BITS)


@@ -456,6 +431,15 @@ asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode
                                (__START_KERNEL & PGDIR_MASK)));
        BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
 
                                (__START_KERNEL & PGDIR_MASK)));
        BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
 

+       if (check_la57_support()) {
+               __pgtable_l5_enabled    = 1;
+               pgdir_shift             = 48;
+               ptrs_per_p4d            = 512;
+               page_offset_base        = __PAGE_OFFSET_BASE_L5;
+               vmalloc_base            = __VMALLOC_BASE_L5;
+               vmemmap_base            = __VMEMMAP_BASE_L5;
+       }
+

        cr4_init_shadow();
 
        /* Kill off the identity-map trampoline */
        cr4_init_shadow();
 
        /* Kill off the identity-map trampoline */