summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/Kconfig12
-rw-r--r--arch/x86/Makefile2
-rw-r--r--arch/x86/boot/compressed/tdx.c6
-rw-r--r--arch/x86/coco/tdx/tdcall.S234
-rw-r--r--arch/x86/coco/tdx/tdx-shared.c28
-rw-r--r--arch/x86/coco/tdx/tdx.c138
-rw-r--r--arch/x86/hyperv/ivm.c10
-rw-r--r--arch/x86/include/asm/shared/tdx.h89
-rw-r--r--arch/x86/include/asm/tdx.h11
-rw-r--r--arch/x86/kernel/asm-offsets.c33
-rw-r--r--arch/x86/virt/Makefile2
-rw-r--r--arch/x86/virt/vmx/Makefile2
-rw-r--r--arch/x86/virt/vmx/tdx/Makefile2
-rw-r--r--arch/x86/virt/vmx/tdx/seamcall.S61
-rw-r--r--arch/x86/virt/vmx/tdx/tdxcall.S226
-rw-r--r--tools/objtool/noreturns.h1
16 files changed, 491 insertions, 366 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index ad478a2b49e2..433f5e1906d1 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -1940,6 +1940,18 @@ config X86_USER_SHADOW_STACK
If unsure, say N.
+config INTEL_TDX_HOST
+ bool "Intel Trust Domain Extensions (TDX) host support"
+ depends on CPU_SUP_INTEL
+ depends on X86_64
+ depends on KVM_INTEL
+ help
+ Intel Trust Domain Extensions (TDX) protects guest VMs from malicious
+ host and certain physical attacks. This option enables necessary TDX
+ support in the host kernel to run confidential VMs.
+
+ If unsure, say N.
+
config EFI
bool "EFI runtime service support"
depends on ACPI
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 76da1e8b3eb0..4de6ddaf4b84 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -253,6 +253,8 @@ archheaders:
libs-y += arch/x86/lib/
+core-y += arch/x86/virt/
+
# drivers-y are linked after core-y
drivers-$(CONFIG_MATH_EMULATION) += arch/x86/math-emu/
drivers-$(CONFIG_PCI) += arch/x86/pci/
diff --git a/arch/x86/boot/compressed/tdx.c b/arch/x86/boot/compressed/tdx.c
index 8841b945a1e2..8451d6a1030c 100644
--- a/arch/x86/boot/compressed/tdx.c
+++ b/arch/x86/boot/compressed/tdx.c
@@ -18,7 +18,7 @@ void __tdx_hypercall_failed(void)
static inline unsigned int tdx_io_in(int size, u16 port)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),
.r12 = size,
@@ -26,7 +26,7 @@ static inline unsigned int tdx_io_in(int size, u16 port)
.r14 = port,
};
- if (__tdx_hypercall_ret(&args))
+ if (__tdx_hypercall(&args))
return UINT_MAX;
return args.r11;
@@ -34,7 +34,7 @@ static inline unsigned int tdx_io_in(int size, u16 port)
static inline void tdx_io_out(int size, u16 port, u32 value)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),
.r12 = size,
diff --git a/arch/x86/coco/tdx/tdcall.S b/arch/x86/coco/tdx/tdcall.S
index b193c0a1d8db..52d9786da308 100644
--- a/arch/x86/coco/tdx/tdcall.S
+++ b/arch/x86/coco/tdx/tdcall.S
@@ -1,239 +1,63 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <asm/asm-offsets.h>
#include <asm/asm.h>
-#include <asm/frame.h>
-#include <asm/unwind_hints.h>
#include <linux/linkage.h>
-#include <linux/bits.h>
#include <linux/errno.h>
#include "../../virt/vmx/tdx/tdxcall.S"
-/*
- * Bitmasks of exposed registers (with VMM).
- */
-#define TDX_RDX BIT(2)
-#define TDX_RBX BIT(3)
-#define TDX_RSI BIT(6)
-#define TDX_RDI BIT(7)
-#define TDX_R8 BIT(8)
-#define TDX_R9 BIT(9)
-#define TDX_R10 BIT(10)
-#define TDX_R11 BIT(11)
-#define TDX_R12 BIT(12)
-#define TDX_R13 BIT(13)
-#define TDX_R14 BIT(14)
-#define TDX_R15 BIT(15)
-
-/*
- * These registers are clobbered to hold arguments for each
- * TDVMCALL. They are safe to expose to the VMM.
- * Each bit in this mask represents a register ID. Bit field
- * details can be found in TDX GHCI specification, section
- * titled "TDCALL [TDG.VP.VMCALL] leaf".
- */
-#define TDVMCALL_EXPOSE_REGS_MASK \
- ( TDX_RDX | TDX_RBX | TDX_RSI | TDX_RDI | TDX_R8 | TDX_R9 | \
- TDX_R10 | TDX_R11 | TDX_R12 | TDX_R13 | TDX_R14 | TDX_R15 )
-
.section .noinstr.text, "ax"
/*
- * __tdx_module_call() - Used by TDX guests to request services from
- * the TDX module (does not include VMM services) using TDCALL instruction.
- *
- * Transforms function call register arguments into the TDCALL register ABI.
- * After TDCALL operation, TDX module output is saved in @out (if it is
- * provided by the user).
- *
- *-------------------------------------------------------------------------
- * TDCALL ABI:
- *-------------------------------------------------------------------------
- * Input Registers:
- *
- * RAX - TDCALL Leaf number.
- * RCX,RDX,R8-R9 - TDCALL Leaf specific input registers.
- *
- * Output Registers:
+ * __tdcall() - Used by TDX guests to request services from the TDX
+ * module (does not include VMM services) using TDCALL instruction.
*
- * RAX - TDCALL instruction error code.
- * RCX,RDX,R8-R11 - TDCALL Leaf specific output registers.
+ * __tdcall() function ABI:
*
- *-------------------------------------------------------------------------
+ * @fn (RDI) - TDCALL Leaf ID, moved to RAX
+ * @args (RSI) - struct tdx_module_args for input
*
- * __tdx_module_call() function ABI:
- *
- * @fn (RDI) - TDCALL Leaf ID, moved to RAX
- * @rcx (RSI) - Input parameter 1, moved to RCX
- * @rdx (RDX) - Input parameter 2, moved to RDX
- * @r8 (RCX) - Input parameter 3, moved to R8
- * @r9 (R8) - Input parameter 4, moved to R9
- *
- * @out (R9) - struct tdx_module_output pointer
- * stored temporarily in R12 (not
- * shared with the TDX module). It
- * can be NULL.
+ * Only RCX/RDX/R8-R11 are used as input registers.
*
* Return status of TDCALL via RAX.
*/
-SYM_FUNC_START(__tdx_module_call)
- FRAME_BEGIN
+SYM_FUNC_START(__tdcall)
TDX_MODULE_CALL host=0
- FRAME_END
- RET
-SYM_FUNC_END(__tdx_module_call)
+SYM_FUNC_END(__tdcall)
/*
- * TDX_HYPERCALL - Make hypercalls to a TDX VMM using TDVMCALL leaf of TDCALL
- * instruction
- *
- * Transforms values in function call argument struct tdx_hypercall_args @args
- * into the TDCALL register ABI. After TDCALL operation, VMM output is saved
- * back in @args, if \ret is 1.
- *
- *-------------------------------------------------------------------------
- * TD VMCALL ABI:
- *-------------------------------------------------------------------------
- *
- * Input Registers:
+ * __tdcall_ret() - Used by TDX guests to request services from the TDX
+ * module (does not include VMM services) using TDCALL instruction, with
+ * saving output registers to the 'struct tdx_module_args' used as input.
*
- * RAX - TDCALL instruction leaf number (0 - TDG.VP.VMCALL)
- * RCX - BITMAP which controls which part of TD Guest GPR
- * is passed as-is to the VMM and back.
- * R10 - Set 0 to indicate TDCALL follows standard TDX ABI
- * specification. Non zero value indicates vendor
- * specific ABI.
- * R11 - VMCALL sub function number
- * RBX, RDX, RDI, RSI - Used to pass VMCALL sub function specific arguments.
- * R8-R9, R12-R15 - Same as above.
+ * __tdcall_ret() function ABI:
*
- * Output Registers:
+ * @fn (RDI) - TDCALL Leaf ID, moved to RAX
+ * @args (RSI) - struct tdx_module_args for input and output
*
- * RAX - TDCALL instruction status (Not related to hypercall
- * output).
- * RBX, RDX, RDI, RSI - Hypercall sub function specific output values.
- * R8-R15 - Same as above.
+ * Only RCX/RDX/R8-R11 are used as input/output registers.
*
+ * Return status of TDCALL via RAX.
*/
-.macro TDX_HYPERCALL ret:req
- FRAME_BEGIN
-
- /* Save callee-saved GPRs as mandated by the x86_64 ABI */
- push %r15
- push %r14
- push %r13
- push %r12
- push %rbx
-
- /* Free RDI to be used as TDVMCALL arguments */
- movq %rdi, %rax
-
- /* Copy hypercall registers from arg struct: */
- movq TDX_HYPERCALL_r8(%rax), %r8
- movq TDX_HYPERCALL_r9(%rax), %r9
- movq TDX_HYPERCALL_r10(%rax), %r10
- movq TDX_HYPERCALL_r11(%rax), %r11
- movq TDX_HYPERCALL_r12(%rax), %r12
- movq TDX_HYPERCALL_r13(%rax), %r13
- movq TDX_HYPERCALL_r14(%rax), %r14
- movq TDX_HYPERCALL_r15(%rax), %r15
- movq TDX_HYPERCALL_rdi(%rax), %rdi
- movq TDX_HYPERCALL_rsi(%rax), %rsi
- movq TDX_HYPERCALL_rbx(%rax), %rbx
- movq TDX_HYPERCALL_rdx(%rax), %rdx
-
- push %rax
-
- /* Mangle function call ABI into TDCALL ABI: */
- /* Set TDCALL leaf ID (TDVMCALL (0)) in RAX */
- xor %eax, %eax
-
- movl $TDVMCALL_EXPOSE_REGS_MASK, %ecx
-
- tdcall
-
- /*
- * RAX!=0 indicates a failure of the TDVMCALL mechanism itself and that
- * something has gone horribly wrong with the TDX module.
- *
- * The return status of the hypercall operation is in a separate
- * register (in R10). Hypercall errors are a part of normal operation
- * and are handled by callers.
- */
- testq %rax, %rax
- jne .Lpanic\@
-
- pop %rax
-
- .if \ret
- movq %r8, TDX_HYPERCALL_r8(%rax)
- movq %r9, TDX_HYPERCALL_r9(%rax)
- movq %r10, TDX_HYPERCALL_r10(%rax)
- movq %r11, TDX_HYPERCALL_r11(%rax)
- movq %r12, TDX_HYPERCALL_r12(%rax)
- movq %r13, TDX_HYPERCALL_r13(%rax)
- movq %r14, TDX_HYPERCALL_r14(%rax)
- movq %r15, TDX_HYPERCALL_r15(%rax)
- movq %rdi, TDX_HYPERCALL_rdi(%rax)
- movq %rsi, TDX_HYPERCALL_rsi(%rax)
- movq %rbx, TDX_HYPERCALL_rbx(%rax)
- movq %rdx, TDX_HYPERCALL_rdx(%rax)
- .endif
-
- /* TDVMCALL leaf return code is in R10 */
- movq %r10, %rax
-
- /*
- * Zero out registers exposed to the VMM to avoid speculative execution
- * with VMM-controlled values. This needs to include all registers
- * present in TDVMCALL_EXPOSE_REGS_MASK, except RBX, and R12-R15 which
- * will be restored.
- */
- xor %r8d, %r8d
- xor %r9d, %r9d
- xor %r10d, %r10d
- xor %r11d, %r11d
- xor %rdi, %rdi
- xor %rdx, %rdx
-
- /* Restore callee-saved GPRs as mandated by the x86_64 ABI */
- pop %rbx
- pop %r12
- pop %r13
- pop %r14
- pop %r15
-
- FRAME_END
-
- RET
-.Lpanic\@:
- call __tdx_hypercall_failed
- /* __tdx_hypercall_failed never returns */
- REACHABLE
- jmp .Lpanic\@
-.endm
+SYM_FUNC_START(__tdcall_ret)
+ TDX_MODULE_CALL host=0 ret=1
+SYM_FUNC_END(__tdcall_ret)
/*
+ * __tdcall_saved_ret() - Used by TDX guests to request services from the
+ * TDX module (including VMM services) using TDCALL instruction, with
+ * saving output registers to the 'struct tdx_module_args' used as input.
*
- * __tdx_hypercall() function ABI:
- *
- * @args (RDI) - struct tdx_hypercall_args for input
- *
- * On successful completion, return the hypercall error code.
- */
-SYM_FUNC_START(__tdx_hypercall)
- TDX_HYPERCALL ret=0
-SYM_FUNC_END(__tdx_hypercall)
-
-/*
+ * __tdcall_saved_ret() function ABI:
*
- * __tdx_hypercall_ret() function ABI:
+ * @fn (RDI) - TDCALL leaf ID, moved to RAX
+ * @args (RSI) - struct tdx_module_args for input/output
*
- * @args (RDI) - struct tdx_hypercall_args for input and output
+ * All registers in @args are used as input/output registers.
*
* On successful completion, return the hypercall error code.
*/
-SYM_FUNC_START(__tdx_hypercall_ret)
- TDX_HYPERCALL ret=1
-SYM_FUNC_END(__tdx_hypercall_ret)
+SYM_FUNC_START(__tdcall_saved_ret)
+ TDX_MODULE_CALL host=0 ret=1 saved=1
+SYM_FUNC_END(__tdcall_saved_ret)
diff --git a/arch/x86/coco/tdx/tdx-shared.c b/arch/x86/coco/tdx/tdx-shared.c
index ef20ddc37b58..78e413269791 100644
--- a/arch/x86/coco/tdx/tdx-shared.c
+++ b/arch/x86/coco/tdx/tdx-shared.c
@@ -5,7 +5,7 @@ static unsigned long try_accept_one(phys_addr_t start, unsigned long len,
enum pg_level pg_level)
{
unsigned long accept_size = page_level_size(pg_level);
- u64 tdcall_rcx;
+ struct tdx_module_args args = {};
u8 page_size;
if (!IS_ALIGNED(start, accept_size))
@@ -34,8 +34,8 @@ static unsigned long try_accept_one(phys_addr_t start, unsigned long len,
return 0;
}
- tdcall_rcx = start | page_size;
- if (__tdx_module_call(TDX_ACCEPT_PAGE, tdcall_rcx, 0, 0, 0, NULL))
+ args.rcx = start | page_size;
+ if (__tdcall(TDG_MEM_PAGE_ACCEPT, &args))
return 0;
return accept_size;
@@ -45,7 +45,7 @@ bool tdx_accept_memory(phys_addr_t start, phys_addr_t end)
{
/*
* For shared->private conversion, accept the page using
- * TDX_ACCEPT_PAGE TDX module call.
+ * TDG_MEM_PAGE_ACCEPT TDX module call.
*/
while (start < end) {
unsigned long len = end - start;
@@ -69,3 +69,23 @@ bool tdx_accept_memory(phys_addr_t start, phys_addr_t end)
return true;
}
+
+noinstr u64 __tdx_hypercall(struct tdx_module_args *args)
+{
+ /*
+ * For TDVMCALL explicitly set RCX to the bitmap of shared registers.
+ * The caller isn't expected to set @args->rcx anyway.
+ */
+ args->rcx = TDVMCALL_EXPOSE_REGS_MASK;
+
+ /*
+ * Failure of __tdcall_saved_ret() indicates a failure of the TDVMCALL
+ * mechanism itself and that something has gone horribly wrong with
+ * the TDX module. __tdx_hypercall_failed() never returns.
+ */
+ if (__tdcall_saved_ret(TDG_VP_VMCALL, args))
+ __tdx_hypercall_failed();
+
+ /* TDVMCALL leaf return code is in R10 */
+ return args->r10;
+}
diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c
index 2e1be592c220..d11206ceff3b 100644
--- a/arch/x86/coco/tdx/tdx.c
+++ b/arch/x86/coco/tdx/tdx.c
@@ -38,7 +38,7 @@
#define TDREPORT_SUBTYPE_0 0
/* Called from __tdx_hypercall() for unrecoverable failure */
-noinstr void __tdx_hypercall_failed(void)
+noinstr void __noreturn __tdx_hypercall_failed(void)
{
instrumentation_begin();
panic("TDVMCALL failed. TDX module bug?");
@@ -48,7 +48,7 @@ noinstr void __tdx_hypercall_failed(void)
long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2,
unsigned long p3, unsigned long p4)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = nr,
.r11 = p1,
.r12 = p2,
@@ -66,10 +66,9 @@ EXPORT_SYMBOL_GPL(tdx_kvm_hypercall);
* should only be used for calls that have no legitimate reason to fail
* or where the kernel can not survive the call failing.
*/
-static inline void tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9,
- struct tdx_module_output *out)
+static inline void tdcall(u64 fn, struct tdx_module_args *args)
{
- if (__tdx_module_call(fn, rcx, rdx, r8, r9, out))
+ if (__tdcall_ret(fn, args))
panic("TDCALL %lld failed (Buggy TDX module!)\n", fn);
}
@@ -89,11 +88,14 @@ static inline void tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9,
*/
int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
{
+ struct tdx_module_args args = {
+ .rcx = virt_to_phys(tdreport),
+ .rdx = virt_to_phys(reportdata),
+ .r8 = TDREPORT_SUBTYPE_0,
+ };
u64 ret;
- ret = __tdx_module_call(TDX_GET_REPORT, virt_to_phys(tdreport),
- virt_to_phys(reportdata), TDREPORT_SUBTYPE_0,
- 0, NULL);
+ ret = __tdcall(TDG_MR_REPORT, &args);
if (ret) {
if (TDCALL_RETURN_CODE(ret) == TDCALL_INVALID_OPERAND)
return -EINVAL;
@@ -106,7 +108,7 @@ EXPORT_SYMBOL_GPL(tdx_mcall_get_report0);
static void __noreturn tdx_panic(const char *msg)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = TDVMCALL_REPORT_FATAL_ERROR,
.r12 = 0, /* Error code: 0 is Panic */
@@ -141,7 +143,7 @@ static void __noreturn tdx_panic(const char *msg)
static void tdx_parse_tdinfo(u64 *cc_mask)
{
- struct tdx_module_output out;
+ struct tdx_module_args args = {};
unsigned int gpa_width;
u64 td_attr;
@@ -152,7 +154,7 @@ static void tdx_parse_tdinfo(u64 *cc_mask)
* Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL
* [TDG.VP.INFO].
*/
- tdx_module_call(TDX_GET_INFO, 0, 0, 0, 0, &out);
+ tdcall(TDG_VP_INFO, &args);
/*
* The highest bit of a guest physical address is the "sharing" bit.
@@ -161,7 +163,7 @@ static void tdx_parse_tdinfo(u64 *cc_mask)
* The GPA width that comes out of this call is critical. TDX guests
* can not meaningfully run without it.
*/
- gpa_width = out.rcx & GENMASK(5, 0);
+ gpa_width = args.rcx & GENMASK(5, 0);
*cc_mask = BIT_ULL(gpa_width - 1);
/*
@@ -169,7 +171,7 @@ static void tdx_parse_tdinfo(u64 *cc_mask)
* memory. Ensure that no #VE will be delivered for accesses to
* TD-private memory. Only VMM-shared memory (MMIO) will #VE.
*/
- td_attr = out.rdx;
+ td_attr = args.rdx;
if (!(td_attr & ATTR_SEPT_VE_DISABLE)) {
const char *msg = "TD misconfiguration: SEPT_VE_DISABLE attribute must be set.";
@@ -228,7 +230,7 @@ static int ve_instr_len(struct ve_info *ve)
static u64 __cpuidle __halt(const bool irq_disabled)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_HLT),
.r12 = irq_disabled,
@@ -272,7 +274,7 @@ void __cpuidle tdx_safe_halt(void)
static int read_msr(struct pt_regs *regs, struct ve_info *ve)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_MSR_READ),
.r12 = regs->cx,
@@ -283,7 +285,7 @@ static int read_msr(struct pt_regs *regs, struct ve_info *ve)
* can be found in TDX Guest-Host-Communication Interface
* (GHCI), section titled "TDG.VP.VMCALL<Instruction.RDMSR>".
*/
- if (__tdx_hypercall_ret(&args))
+ if (__tdx_hypercall(&args))
return -EIO;
regs->ax = lower_32_bits(args.r11);
@@ -293,7 +295,7 @@ static int read_msr(struct pt_regs *regs, struct ve_info *ve)
static int write_msr(struct pt_regs *regs, struct ve_info *ve)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_MSR_WRITE),
.r12 = regs->cx,
@@ -313,7 +315,7 @@ static int write_msr(struct pt_regs *regs, struct ve_info *ve)
static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_CPUID),
.r12 = regs->ax,
@@ -337,7 +339,7 @@ static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
* ABI can be found in TDX Guest-Host-Communication Interface
* (GHCI), section titled "VP.VMCALL<Instruction.CPUID>".
*/
- if (__tdx_hypercall_ret(&args))
+ if (__tdx_hypercall(&args))
return -EIO;
/*
@@ -355,7 +357,7 @@ static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
static bool mmio_read(int size, unsigned long addr, unsigned long *val)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_EPT_VIOLATION),
.r12 = size,
@@ -364,8 +366,9 @@ static bool mmio_read(int size, unsigned long addr, unsigned long *val)
.r15 = *val,
};
- if (__tdx_hypercall_ret(&args))
+ if (__tdx_hypercall(&args))
return false;
+
*val = args.r11;
return true;
}
@@ -483,7 +486,7 @@ static int handle_mmio(struct pt_regs *regs, struct ve_info *ve)
static bool handle_in(struct pt_regs *regs, int size, int port)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = hcall_func(EXIT_REASON_IO_INSTRUCTION),
.r12 = size,
@@ -498,7 +501,7 @@ static bool handle_in(struct pt_regs *regs, int size, int port)
* in TDX Guest-Host-Communication Interface (GHCI) section titled
* "TDG.VP.VMCALL<Instruction.IO>".
*/
- success = !__tdx_hypercall_ret(&args);
+ success = !__tdx_hypercall(&args);
/* Update part of the register affected by the emulated instruction */
regs->ax &= ~mask;
@@ -577,7 +580,7 @@ __init bool tdx_early_handle_ve(struct pt_regs *regs)
void tdx_get_ve_info(struct ve_info *ve)
{
- struct tdx_module_output out;
+ struct tdx_module_args args = {};
/*
* Called during #VE handling to retrieve the #VE info from the
@@ -594,15 +597,15 @@ void tdx_get_ve_info(struct ve_info *ve)
* Note, the TDX module treats virtual NMIs as inhibited if the #VE
* valid flag is set. It means that NMI=>#VE will not result in a #DF.
*/
- tdx_module_call(TDX_GET_VEINFO, 0, 0, 0, 0, &out);
+ tdcall(TDG_VP_VEINFO_GET, &args);
/* Transfer the output parameters */
- ve->exit_reason = out.rcx;
- ve->exit_qual = out.rdx;
- ve->gla = out.r8;
- ve->gpa = out.r9;
- ve->instr_len = lower_32_bits(out.r10);
- ve->instr_info = upper_32_bits(out.r10);
+ ve->exit_reason = args.rcx;
+ ve->exit_qual = args.rdx;
+ ve->gla = args.r8;
+ ve->gpa = args.r9;
+ ve->instr_len = lower_32_bits(args.r10);
+ ve->instr_info = upper_32_bits(args.r10);
}
/*
@@ -703,14 +706,15 @@ static bool tdx_cache_flush_required(void)
}
/*
- * Inform the VMM of the guest's intent for this physical page: shared with
- * the VMM or private to the guest. The VMM is expected to change its mapping
- * of the page in response.
+ * Notify the VMM about page mapping conversion. More info about ABI
+ * can be found in TDX Guest-Host-Communication Interface (GHCI),
+ * section "TDG.VP.VMCALL<MapGPA>".
*/
-static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)
+static bool tdx_map_gpa(phys_addr_t start, phys_addr_t end, bool enc)
{
- phys_addr_t start = __pa(vaddr);
- phys_addr_t end = __pa(vaddr + numpages * PAGE_SIZE);
+ /* Retrying the hypercall a second time should succeed; use 3 just in case */
+ const int max_retries_per_page = 3;
+ int retry_count = 0;
if (!enc) {
/* Set the shared (decrypted) bits: */
@@ -718,12 +722,51 @@ static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)
end |= cc_mkdec(0);
}
- /*
- * Notify the VMM about page mapping conversion. More info about ABI
- * can be found in TDX Guest-Host-Communication Interface (GHCI),
- * section "TDG.VP.VMCALL<MapGPA>"
- */
- if (_tdx_hypercall(TDVMCALL_MAP_GPA, start, end - start, 0, 0))
+ while (retry_count < max_retries_per_page) {
+ struct tdx_module_args args = {
+ .r10 = TDX_HYPERCALL_STANDARD,
+ .r11 = TDVMCALL_MAP_GPA,
+ .r12 = start,
+ .r13 = end - start };
+
+ u64 map_fail_paddr;
+ u64 ret = __tdx_hypercall(&args);
+
+ if (ret != TDVMCALL_STATUS_RETRY)
+ return !ret;
+ /*
+ * The guest must retry the operation for the pages in the
+ * region starting at the GPA specified in R11. R11 comes
+ * from the untrusted VMM. Sanity check it.
+ */
+ map_fail_paddr = args.r11;
+ if (map_fail_paddr < start || map_fail_paddr >= end)
+ return false;
+
+ /* "Consume" a retry without forward progress */
+ if (map_fail_paddr == start) {
+ retry_count++;
+ continue;
+ }
+
+ start = map_fail_paddr;
+ retry_count = 0;
+ }
+
+ return false;
+}
+
+/*
+ * Inform the VMM of the guest's intent for this physical page: shared with
+ * the VMM or private to the guest. The VMM is expected to change its mapping
+ * of the page in response.
+ */
+static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)
+{
+ phys_addr_t start = __pa(vaddr);
+ phys_addr_t end = __pa(vaddr + numpages * PAGE_SIZE);
+
+ if (!tdx_map_gpa(start, end, enc))
return false;
/* shared->private conversion requires memory to be accepted before use */
@@ -759,6 +802,10 @@ static bool tdx_enc_status_change_finish(unsigned long vaddr, int numpages,
void __init tdx_early_init(void)
{
+ struct tdx_module_args args = {
+ .rdx = TDCS_NOTIFY_ENABLES,
+ .r9 = -1ULL,
+ };
u64 cc_mask;
u32 eax, sig[3];
@@ -769,12 +816,15 @@ void __init tdx_early_init(void)
setup_force_cpu_cap(X86_FEATURE_TDX_GUEST);
+ /* TSC is the only reliable clock in TDX guest */
+ setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
+
cc_vendor = CC_VENDOR_INTEL;
tdx_parse_tdinfo(&cc_mask);
cc_set_mask(cc_mask);
/* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */
- tdx_module_call(TDX_WR, 0, TDCS_NOTIFY_ENABLES, 0, -1ULL, NULL);
+ tdcall(TDG_VM_WR, &args);
/*
* All bits above GPA width are reserved and kernel treats shared bit
diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c
index c6edde1a1dec..02e55237d919 100644
--- a/arch/x86/hyperv/ivm.c
+++ b/arch/x86/hyperv/ivm.c
@@ -384,7 +384,7 @@ static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {}
#ifdef CONFIG_INTEL_TDX_GUEST
static void hv_tdx_msr_write(u64 msr, u64 val)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = EXIT_REASON_MSR_WRITE,
.r12 = msr,
@@ -398,13 +398,13 @@ static void hv_tdx_msr_write(u64 msr, u64 val)
static void hv_tdx_msr_read(u64 msr, u64 *val)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = EXIT_REASON_MSR_READ,
.r12 = msr,
};
- u64 ret = __tdx_hypercall_ret(&args);
+ u64 ret = __tdx_hypercall(&args);
if (WARN_ONCE(ret, "Failed to emulate MSR read: %lld\n", ret))
*val = 0;
@@ -414,13 +414,13 @@ static void hv_tdx_msr_read(u64 msr, u64 *val)
u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2)
{
- struct tdx_hypercall_args args = { };
+ struct tdx_module_args args = { };
args.r10 = control;
args.rdx = param1;
args.r8 = param2;
- (void)__tdx_hypercall_ret(&args);
+ (void)__tdx_hypercall(&args);
return args.r11;
}
diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h
index 7513b3bb69b7..f74695dea217 100644
--- a/arch/x86/include/asm/shared/tdx.h
+++ b/arch/x86/include/asm/shared/tdx.h
@@ -11,11 +11,12 @@
#define TDX_IDENT "IntelTDX "
/* TDX module Call Leaf IDs */
-#define TDX_GET_INFO 1
-#define TDX_GET_VEINFO 3
-#define TDX_GET_REPORT 4
-#define TDX_ACCEPT_PAGE 6
-#define TDX_WR 8
+#define TDG_VP_VMCALL 0
+#define TDG_VP_INFO 1
+#define TDG_VP_VEINFO_GET 3
+#define TDG_MR_REPORT 4
+#define TDG_MEM_PAGE_ACCEPT 6
+#define TDG_VM_WR 8
/* TDCS fields. To be used by TDG.VM.WR and TDG.VM.RD module calls */
#define TDCS_NOTIFY_ENABLES 0x9100000000000010
@@ -24,32 +25,70 @@
#define TDVMCALL_MAP_GPA 0x10001
#define TDVMCALL_REPORT_FATAL_ERROR 0x10003
+#define TDVMCALL_STATUS_RETRY 1
+
+/*
+ * Bitmasks of exposed registers (with VMM).
+ */
+#define TDX_RDX BIT(2)
+#define TDX_RBX BIT(3)
+#define TDX_RSI BIT(6)
+#define TDX_RDI BIT(7)
+#define TDX_R8 BIT(8)
+#define TDX_R9 BIT(9)
+#define TDX_R10 BIT(10)
+#define TDX_R11 BIT(11)
+#define TDX_R12 BIT(12)
+#define TDX_R13 BIT(13)
+#define TDX_R14 BIT(14)
+#define TDX_R15 BIT(15)
+
+/*
+ * These registers are clobbered to hold arguments for each
+ * TDVMCALL. They are safe to expose to the VMM.
+ * Each bit in this mask represents a register ID. Bit field
+ * details can be found in TDX GHCI specification, section
+ * titled "TDCALL [TDG.VP.VMCALL] leaf".
+ */
+#define TDVMCALL_EXPOSE_REGS_MASK \
+ (TDX_RDX | TDX_RBX | TDX_RSI | TDX_RDI | TDX_R8 | TDX_R9 | \
+ TDX_R10 | TDX_R11 | TDX_R12 | TDX_R13 | TDX_R14 | TDX_R15)
+
#ifndef __ASSEMBLY__
+#include <linux/compiler_attributes.h>
+
/*
- * Used in __tdx_hypercall() to pass down and get back registers' values of
- * the TDCALL instruction when requesting services from the VMM.
- *
- * This is a software only structure and not part of the TDX module/VMM ABI.
+ * Used in __tdcall*() to gather the input/output registers' values of the
+ * TDCALL instruction when requesting services from the TDX module. This is a
+ * software only structure and not part of the TDX module/VMM ABI
*/
-struct tdx_hypercall_args {
+struct tdx_module_args {
+ /* callee-clobbered */
+ u64 rcx;
+ u64 rdx;
u64 r8;
u64 r9;
+ /* extra callee-clobbered */
u64 r10;
u64 r11;
+ /* callee-saved + rdi/rsi */
u64 r12;
u64 r13;
u64 r14;
u64 r15;
+ u64 rbx;
u64 rdi;
u64 rsi;
- u64 rbx;
- u64 rdx;
};
+/* Used to communicate with the TDX module */
+u64 __tdcall(u64 fn, struct tdx_module_args *args);
+u64 __tdcall_ret(u64 fn, struct tdx_module_args *args);
+u64 __tdcall_saved_ret(u64 fn, struct tdx_module_args *args);
+
/* Used to request services from the VMM */
-u64 __tdx_hypercall(struct tdx_hypercall_args *args);
-u64 __tdx_hypercall_ret(struct tdx_hypercall_args *args);
+u64 __tdx_hypercall(struct tdx_module_args *args);
/*
* Wrapper for standard use of __tdx_hypercall with no output aside from
@@ -57,7 +96,7 @@ u64 __tdx_hypercall_ret(struct tdx_hypercall_args *args);
*/
static inline u64 _tdx_hypercall(u64 fn, u64 r12, u64 r13, u64 r14, u64 r15)
{
- struct tdx_hypercall_args args = {
+ struct tdx_module_args args = {
.r10 = TDX_HYPERCALL_STANDARD,
.r11 = fn,
.r12 = r12,
@@ -71,25 +110,7 @@ static inline u64 _tdx_hypercall(u64 fn, u64 r12, u64 r13, u64 r14, u64 r15)
/* Called from __tdx_hypercall() for unrecoverable failure */
-void __tdx_hypercall_failed(void);
-
-/*
- * Used in __tdx_module_call() to gather the output registers' values of the
- * TDCALL instruction when requesting services from the TDX module. This is a
- * software only structure and not part of the TDX module/VMM ABI
- */
-struct tdx_module_output {
- u64 rcx;
- u64 rdx;
- u64 r8;
- u64 r9;
- u64 r10;
- u64 r11;
-};
-
-/* Used to communicate with the TDX module */
-u64 __tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9,
- struct tdx_module_output *out);
+void __noreturn __tdx_hypercall_failed(void);
bool tdx_accept_memory(phys_addr_t start, phys_addr_t end);
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index 603e6d1e9d4a..adcbe3f1de30 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -8,6 +8,7 @@
#include <asm/errno.h>
#include <asm/ptrace.h>
+#include <asm/trapnr.h>
#include <asm/shared/tdx.h>
/*
@@ -20,6 +21,9 @@
#define TDX_SW_ERROR (TDX_ERROR | GENMASK_ULL(47, 40))
#define TDX_SEAMCALL_VMFAILINVALID (TDX_SW_ERROR | _UL(0xFFFF0000))
+#define TDX_SEAMCALL_GP (TDX_SW_ERROR | X86_TRAP_GP)
+#define TDX_SEAMCALL_UD (TDX_SW_ERROR | X86_TRAP_UD)
+
#ifndef __ASSEMBLY__
/*
@@ -72,5 +76,12 @@ static inline long tdx_kvm_hypercall(unsigned int nr, unsigned long p1,
return -ENODEV;
}
#endif /* CONFIG_INTEL_TDX_GUEST && CONFIG_KVM_GUEST */
+
+#ifdef CONFIG_INTEL_TDX_HOST
+u64 __seamcall(u64 fn, struct tdx_module_args *args);
+u64 __seamcall_ret(u64 fn, struct tdx_module_args *args);
+u64 __seamcall_saved_ret(u64 fn, struct tdx_module_args *args);
+#endif /* CONFIG_INTEL_TDX_HOST */
+
#endif /* !__ASSEMBLY__ */
#endif /* _ASM_X86_TDX_H */
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index dc3576303f1a..6913b372ccf7 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -68,26 +68,19 @@ static void __used common(void)
#endif
BLANK();
- OFFSET(TDX_MODULE_rcx, tdx_module_output, rcx);
- OFFSET(TDX_MODULE_rdx, tdx_module_output, rdx);
- OFFSET(TDX_MODULE_r8, tdx_module_output, r8);
- OFFSET(TDX_MODULE_r9, tdx_module_output, r9);
- OFFSET(TDX_MODULE_r10, tdx_module_output, r10);
- OFFSET(TDX_MODULE_r11, tdx_module_output, r11);
-
- BLANK();
- OFFSET(TDX_HYPERCALL_r8, tdx_hypercall_args, r8);
- OFFSET(TDX_HYPERCALL_r9, tdx_hypercall_args, r9);
- OFFSET(TDX_HYPERCALL_r10, tdx_hypercall_args, r10);
- OFF