for Movable pages. "nn[KMGTPE]", "nn%", and "mirror"
are exclusive, so you cannot specify multiple forms.
+ kfence.burst= [MM,KFENCE] The number of additional successive
+ allocations to be attempted through KFENCE for each
+ sample interval.
+ Format: <unsigned integer>
+ Default: 0
+
+ kfence.check_on_panic=
+ [MM,KFENCE] Whether to check all KFENCE-managed objects'
+ canaries on panic.
+ Format: <bool>
+ Default: false
+
+ kfence.deferrable=
+ [MM,KFENCE] Whether to use a deferrable timer to trigger
+ allocations. This avoids forcing CPU wake-ups if the
+ system is idle, at the risk of a less predictable
+ sample interval.
+ Format: <bool>
+ Default: CONFIG_KFENCE_DEFERRABLE
+
+ kfence.sample_interval=
+ [MM,KFENCE] KFENCE's sample interval in milliseconds.
+ Format: <unsigned integer>
+ 0 - Disable KFENCE.
+ >0 - Enabled KFENCE with given sample interval.
+ Default: CONFIG_KFENCE_SAMPLE_INTERVAL
+
+ kfence.skip_covered_thresh=
+ [MM,KFENCE] If pool utilization reaches this threshold
+ (pool usage%), KFENCE limits currently covered
+ allocations of the same source from further filling
+ up the pool.
+ Format: <unsigned integer>
+ Default: 75
+
kgdbdbgp= [KGDB,HW,EARLY] kgdb over EHCI usb debug port.
Format: <Controller#>[,poll interval]
The controller # is the number of the ehci usb debug
Default is Y (on).
+ kvm.enable_pmu=[KVM,X86]
+ If enabled, KVM will virtualize PMU functionality based
+ on the virtual CPU model defined by userspace. This
+ can be overridden on a per-VM basis via
+ KVM_CAP_PMU_CAPABILITY.
+
+ If disabled, KVM will not virtualize PMU functionality,
+ e.g. MSRs, PMCs, PMIs, etc., even if userspace defines
+ a virtual CPU model that contains PMU assets.
+
+ Note, KVM's vPMU support implicitly requires running
+ with an in-kernel local APIC, e.g. to deliver PMIs to
+ the guest. Running without an in-kernel local APIC is
+ not supported, though KVM will allow such a combination
+ (with severely degraded functionality).
+
+ See also enable_mediated_pmu.
+
+ Default is Y (on).
+
kvm.enable_virt_at_load=[KVM,ARM64,LOONGARCH,MIPS,RISCV,X86]
If enabled, KVM will enable virtualization in hardware
when KVM is loaded, and disable virtualization when KVM
If the value is 0 (the default), KVM will pick a period based
on the ratio, such that a page is zapped after 1 hour on average.
+ kvm-{amd,intel}.enable_mediated_pmu=[KVM,AMD,INTEL]
+ If enabled, KVM will provide a mediated virtual PMU,
+ instead of the default perf-based virtual PMU (if
+ kvm.enable_pmu is true and PMU is enumerated via the
+ virtual CPU model).
+
+ With a perf-based vPMU, KVM operates as a user of perf,
+ i.e. emulates guest PMU counters using perf events.
+ KVM-created perf events are managed by perf as regular
+ (guest-only) events, e.g. are scheduled in/out, contend
+ for hardware resources, etc. Using a perf-based vPMU
+ allows guest and host usage of the PMU to co-exist, but
+ incurs non-trivial overhead and can result in silently
+ dropped guest events (due to resource contention).
+
+ With a mediated vPMU, hardware PMU state is context
+ switched around the world switch to/from the guest.
+ KVM mediates which events the guest can utilize, but
+ gives the guest direct access to all other PMU assets
+ when possible (KVM may intercept some accesses if the
+ virtual CPU model provides a subset of hardware PMU
+ functionality). Using a mediated vPMU significantly
+ reduces PMU virtualization overhead and eliminates lost
+ guest events, but is mutually exclusive with using perf
+ to profile KVM guests and adds latency to most VM-Exits
+ (to context switch PMU state).
+
+ Default is N (off).
+
kvm-amd.nested= [KVM,AMD] Control nested virtualization feature in
KVM/SVM. Default is 1 (enabled).
If there are multiple matching configurations changing
the same attribute, the last one is used.
+ liveupdate= [KNL,EARLY]
+ Format: <bool>
+ Enable Live Update Orchestrator (LUO).
+ Default: off.
+
load_ramdisk= [RAM] [Deprecated]
lockd.nlm_grace_period=P [NFS] Assign grace period.
#include <asm/kvm_pkvm.h>
#include <asm/kvm_ptrauth.h>
#include <asm/sections.h>
+#include <asm/stacktrace/nvhe.h>
#include <kvm/arm_hypercalls.h>
#include <kvm/arm_pmu.h>
static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
+/*
+ * Tracks KVM IOCTLs and their associated KVM capabilities.
+ */
+struct kvm_ioctl_cap_map {
+ unsigned int ioctl;
+ long ext;
+};
+
+/* Make KVM_CAP_NR_VCPUS the reference for features we always supported */
+#define KVM_CAP_ARM_BASIC KVM_CAP_NR_VCPUS
+
+/*
+ * Sorted by ioctl to allow for potential binary search,
+ * though linear scan is sufficient for this size.
+ */
+static const struct kvm_ioctl_cap_map vm_ioctl_caps[] = {
+ { KVM_CREATE_IRQCHIP, KVM_CAP_IRQCHIP },
+ { KVM_ARM_SET_DEVICE_ADDR, KVM_CAP_ARM_SET_DEVICE_ADDR },
+ { KVM_ARM_MTE_COPY_TAGS, KVM_CAP_ARM_MTE },
+ { KVM_SET_DEVICE_ATTR, KVM_CAP_DEVICE_CTRL },
+ { KVM_GET_DEVICE_ATTR, KVM_CAP_DEVICE_CTRL },
+ { KVM_HAS_DEVICE_ATTR, KVM_CAP_DEVICE_CTRL },
+ { KVM_ARM_SET_COUNTER_OFFSET, KVM_CAP_COUNTER_OFFSET },
+ { KVM_ARM_GET_REG_WRITABLE_MASKS, KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES },
+ { KVM_ARM_PREFERRED_TARGET, KVM_CAP_ARM_BASIC },
+};
+
+/*
+ * Set *ext to the capability.
+ * Return 0 if found, or -EINVAL if no IOCTL matches.
+ */
+long kvm_get_cap_for_kvm_ioctl(unsigned int ioctl, long *ext)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(vm_ioctl_caps); i++) {
+ if (vm_ioctl_caps[i].ioctl == ioctl) {
+ *ext = vm_ioctl_caps[i].ext;
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
if (cap->flags)
return -EINVAL;
- if (kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(cap->cap))
+ if (is_protected_kvm_enabled() && !kvm_pkvm_ext_allowed(kvm, cap->cap))
return -EINVAL;
switch (cap->cap) {
{
int r;
- if (kvm && kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(ext))
+ if (is_protected_kvm_enabled() && !kvm_pkvm_ext_allowed(kvm, ext))
return 0;
switch (ext) {
return kvm_wfi_trap_policy == KVM_WFX_NOTRAP;
return single_task_running() &&
+ vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&
(atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
vcpu->kvm->arch.vgic.nassgireq);
}
void __user *argp = (void __user *)arg;
struct kvm_device_attr attr;
+ if (is_protected_kvm_enabled() && !kvm_pkvm_ioctl_allowed(kvm, ioctl))
+ return -EINVAL;
+
switch (ioctl) {
case KVM_CREATE_IRQCHIP: {
int ret;
params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
else
params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
+
+ if (system_supports_mte())
+ params->hcr_el2 |= HCR_ATA;
+ else
+ params->hcr_el2 |= HCR_TID5;
+
if (cpus_have_final_cap(ARM64_KVM_HVHE))
params->hcr_el2 |= HCR_E2H;
params->vttbr = params->vtcr = 0;
if (err)
goto out;
- kvm_register_perf_callbacks(NULL);
+ kvm_register_perf_callbacks();
out:
if (err)
/* Inits Hyp-mode on all online CPUs */
static int __init init_hyp_mode(void)
{
- u32 hyp_va_bits;
+ u32 hyp_va_bits = kvm_hyp_va_bits();
int cpu;
int err = -ENOMEM;
/*
* Allocate Hyp PGD and setup Hyp identity mapping
*/
- err = kvm_mmu_init(&hyp_va_bits);
+ err = kvm_mmu_init(hyp_va_bits);
if (err)
goto out_err;
set_gcsr_sw_flag(LOONGARCH_CSR_PERFCNTR2);
set_gcsr_sw_flag(LOONGARCH_CSR_PERFCTRL3);
set_gcsr_sw_flag(LOONGARCH_CSR_PERFCNTR3);
+
+ if (cpu_has_msgint) {
+ set_gcsr_hw_flag(LOONGARCH_CSR_IPR);
+ set_gcsr_hw_flag(LOONGARCH_CSR_ISR0);
+ set_gcsr_hw_flag(LOONGARCH_CSR_ISR1);
+ set_gcsr_hw_flag(LOONGARCH_CSR_ISR2);
+ set_gcsr_hw_flag(LOONGARCH_CSR_ISR3);
+ }
}
static void kvm_update_vpid(struct kvm_vcpu *vcpu, int cpu)
}
kvm_init_gcsr_flag();
- kvm_register_perf_callbacks(NULL);
+ kvm_register_perf_callbacks();
/* Register LoongArch IPI interrupt controller interface. */
ret = kvm_loongarch_register_ipi_device();
/*
* Performance events x86 architecture code
*
- * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
+ * Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
* Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
* Copyright (C) 2009 Jaswinder Singh Rajput
* Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
#include <linux/device.h>
#include <linux/nospec.h>
#include <linux/static_call.h>
+ #include <linux/kvm_types.h>
#include <asm/apic.h>
#include <asm/stacktrace.h>
.pmu = &pmu,
};
+ static DEFINE_PER_CPU(bool, guest_lvtpc_loaded);
+
DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key);
DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
DEFINE_STATIC_KEY_FALSE(perf_is_hybrid);
apic_write(APIC_LVTPC, APIC_DM_NMI);
}
+ #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
+ void perf_load_guest_lvtpc(u32 guest_lvtpc)
+ {
+ u32 masked = guest_lvtpc & APIC_LVT_MASKED;
+
+ apic_write(APIC_LVTPC,
+ APIC_DM_FIXED | PERF_GUEST_MEDIATED_PMI_VECTOR | masked);
+ this_cpu_write(guest_lvtpc_loaded, true);
+ }
+ EXPORT_SYMBOL_FOR_KVM(perf_load_guest_lvtpc);
+
+ void perf_put_guest_lvtpc(void)
+ {
+ this_cpu_write(guest_lvtpc_loaded, false);
+ apic_write(APIC_LVTPC, APIC_DM_NMI);
+ }
+ EXPORT_SYMBOL_FOR_KVM(perf_put_guest_lvtpc);
+ #endif /* CONFIG_PERF_GUEST_MEDIATED_PMU */
+
static int
perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{
u64 finish_clock;
int ret;
+ /*
+ * Ignore all NMIs when the CPU's LVTPC is configured to route PMIs to
+ * PERF_GUEST_MEDIATED_PMI_VECTOR, i.e. when an NMI time can't be due
+ * to a PMI. Attempting to handle a PMI while the guest's context is
+ * loaded will generate false positives and clobber guest state. Note,
+ * the LVTPC is switched to/from the dedicated mediated PMI IRQ vector
+ * while host events are quiesced.
+ */
+ if (this_cpu_read(guest_lvtpc_loaded))
+ return NMI_DONE;
+
/*
* All PMUs/events that share this PMI handler should make sure to
* increment active_events for their events.
cap->version = x86_pmu.version;
cap->num_counters_gp = x86_pmu_num_counters(NULL);
cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL);
- cap->bit_width_gp = x86_pmu.cntval_bits;
- cap->bit_width_fixed = x86_pmu.cntval_bits;
+ cap->bit_width_gp = cap->num_counters_gp ? x86_pmu.cntval_bits : 0;
+ cap->bit_width_fixed = cap->num_counters_fixed ? x86_pmu.cntval_bits : 0;
cap->events_mask = (unsigned int)x86_pmu.events_maskl;
cap->events_mask_len = x86_pmu.events_mask_len;
cap->pebs_ept = x86_pmu.pebs_ept;
+ cap->mediated = !!(pmu.capabilities & PERF_PMU_CAP_MEDIATED_VPMU);
}
EXPORT_SYMBOL_FOR_KVM(perf_get_x86_pmu_capability);
VCPU_EXREG_PDPTR = NR_VCPU_REGS,
VCPU_EXREG_CR0,
+ /*
+ * Alias AMD's ERAPS (not a real register) to CR3 so that common code
+ * can trigger emulation of the RAP (Return Address Predictor) with
+ * minimal support required in common code. Piggyback CR3 as the RAP
+ * is cleared on writes to CR3, i.e. marking CR3 dirty will naturally
+ * mark ERAPS dirty as well.
+ */
VCPU_EXREG_CR3,
+ VCPU_EXREG_ERAPS = VCPU_EXREG_CR3,
VCPU_EXREG_CR4,
VCPU_EXREG_RFLAGS,
VCPU_EXREG_SEGMENTS,
*/
u64 emulated_counter;
u64 eventsel;
+ u64 eventsel_hw;
struct perf_event *perf_event;
struct kvm_vcpu *vcpu;
/*
unsigned nr_arch_fixed_counters;
unsigned available_event_types;
u64 fixed_ctr_ctrl;
+ u64 fixed_ctr_ctrl_hw;
u64 fixed_ctr_ctrl_rsvd;
u64 global_ctrl;
u64 global_status;
CPUID_24_0_EBX,
CPUID_8000_0021_ECX,
CPUID_7_1_ECX,
+ CPUID_1E_1_EAX,
+ CPUID_24_1_ECX,
NR_KVM_CPU_CAPS,
NKVMCAPINTS = NR_KVM_CPU_CAPS - NCAPINTS,
enum kvm_irqchip_mode {
KVM_IRQCHIP_NONE,
+#ifdef CONFIG_KVM_IOAPIC
KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */
+#endif
KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */
};
+enum kvm_suppress_eoi_broadcast_mode {
+ KVM_SUPPRESS_EOI_BROADCAST_QUIRKED, /* Legacy behavior */
+ KVM_SUPPRESS_EOI_BROADCAST_ENABLED, /* Enable Suppress EOI broadcast */
+ KVM_SUPPRESS_EOI_BROADCAST_DISABLED /* Disable Suppress EOI broadcast */
+};
+
struct kvm_x86_msr_filter {
u8 count;
bool default_allow:1;
bool x2apic_format;
bool x2apic_broadcast_quirk_disabled;
+ enum kvm_suppress_eoi_broadcast_mode suppress_eoi_broadcast_mode;
bool has_mapped_host_mmio;
bool guest_can_read_msr_platform_info;
bool bus_lock_detection_enabled;
bool enable_pmu;
+ bool created_mediated_pmu;
u32 notify_window;
u32 notify_vmexit_flags;
irq_stats(j)->kvm_posted_intr_wakeup_ipis);
seq_puts(p, " Posted-interrupt wakeup event\n");
#endif
+ #ifdef CONFIG_GUEST_PERF_EVENTS
+ seq_printf(p, "%*s: ", prec, "VPMI");
+ for_each_online_cpu(j)
+ seq_printf(p, "%10u ",
+ irq_stats(j)->perf_guest_mediated_pmis);
+ seq_puts(p, " Perf Guest Mediated PMI\n");
+ #endif
#ifdef CONFIG_X86_POSTED_MSI
seq_printf(p, "%*s: ", prec, "PMN");
for_each_online_cpu(j)
}
#endif
+ #ifdef CONFIG_GUEST_PERF_EVENTS
+ /*
+ * Handler for PERF_GUEST_MEDIATED_PMI_VECTOR.
+ */
+ DEFINE_IDTENTRY_SYSVEC(sysvec_perf_guest_mediated_pmi_handler)
+ {
+ apic_eoi();
+ inc_irq_stat(perf_guest_mediated_pmis);
+ perf_guest_handle_mediated_pmi();
+ }
+ #endif
+
#if IS_ENABLED(CONFIG_KVM)
static void dummy_handler(void) {}
static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;
/* Posted Interrupt Descriptors for coalesced MSIs to be posted */
DEFINE_PER_CPU_ALIGNED(struct pi_desc, posted_msi_pi_desc);
+static DEFINE_PER_CPU_CACHE_HOT(bool, posted_msi_handler_active);
void intel_posted_msi_init(void)
{
this_cpu_write(posted_msi_pi_desc.ndst, destination);
}
+void intel_ack_posted_msi_irq(struct irq_data *irqd)
+{
+ irq_move_irq(irqd);
+
+ /*
+ * Handle the rare case that irq_retrigger() raised the actual
+ * assigned vector on the target CPU, which means that it was not
+ * invoked via the posted MSI handler below. In that case APIC EOI
+ * is required as otherwise the ISR entry becomes stale and lower
+ * priority interrupts are never going to be delivered after that.
+ *
+ * If the posted handler invoked the device interrupt handler then
+ * the EOI would be premature because it would acknowledge the
+ * posted vector.
+ */
+ if (unlikely(!__this_cpu_read(posted_msi_handler_active)))
+ apic_eoi();
+}
+
static __always_inline bool handle_pending_pir(unsigned long *pir, struct pt_regs *regs)
{
unsigned long pir_copy[NR_PIR_WORDS];
pid = this_cpu_ptr(&posted_msi_pi_desc);
+ /* Mark the handler active for intel_ack_posted_msi_irq() */
+ __this_cpu_write(posted_msi_handler_active, true);
inc_irq_stat(posted_msi_notification_count);
irq_enter();
apic_eoi();
irq_exit();
+ __this_cpu_write(posted_msi_handler_active, false);
set_irq_regs(old_regs);
}
#endif /* X86_POSTED_MSI */
#undef __KVM_X86_PMU_OP
}
- void kvm_init_pmu_capability(const struct kvm_pmu_ops *pmu_ops)
+ void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops)
{
bool is_intel = boot_cpu_data.x86_vendor == X86_VENDOR_INTEL;
int min_nr_gp_ctrs = pmu_ops->MIN_NR_GP_COUNTERS;
enable_pmu = false;
}
+ if (!enable_pmu || !enable_mediated_pmu || !kvm_host_pmu.mediated ||
+ !pmu_ops->is_mediated_pmu_supported(&kvm_host_pmu))
+ enable_mediated_pmu = false;
+
+ if (!enable_mediated_pmu)
+ pmu_ops->write_global_ctrl = NULL;
+
if (!enable_pmu) {
memset(&kvm_pmu_cap, 0, sizeof(kvm_pmu_cap));
return;
perf_get_hw_event_config(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
}
+ void kvm_handle_guest_mediated_pmi(void)
+ {
+ struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
+
+ if (WARN_ON_ONCE(!vcpu || !kvm_vcpu_has_mediated_pmu(vcpu)))
+ return;
+
+ kvm_make_request(KVM_REQ_PMI, vcpu);
+ }
+
static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi)
{
struct kvm_pmu *pmu = pmc_to_pmu(pmc);
void pmc_write_counter(struct kvm_pmc *pmc, u64 val)
{
+ if (kvm_vcpu_has_mediated_pmu(pmc->vcpu)) {
+ pmc->counter = val & pmc_bitmask(pmc);
+ return;
+ }
+
/*
* Drop any unconsumed accumulated counts, the WRMSR is a write, not a
* read-modify-write. Adjust the counter value so that its value is
return is_fixed_event_allowed(filter, pmc->idx);
}
+ static void kvm_mediated_pmu_refresh_event_filter(struct kvm_pmc *pmc)
+ {
+ bool allowed = pmc_is_event_allowed(pmc);
+ struct kvm_pmu *pmu = pmc_to_pmu(pmc);
+
+ if (pmc_is_gp(pmc)) {
+ pmc->eventsel_hw &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
+ if (allowed)
+ pmc->eventsel_hw |= pmc->eventsel &
+ ARCH_PERFMON_EVENTSEL_ENABLE;
+ } else {
+ u64 mask = intel_fixed_bits_by_idx(pmc->idx - KVM_FIXED_PMC_BASE_IDX, 0xf);
+
+ pmu->fixed_ctr_ctrl_hw &= ~mask;
+ if (allowed)
+ pmu->fixed_ctr_ctrl_hw |= pmu->fixed_ctr_ctrl & mask;
+ }
+ }
+
static int reprogram_counter(struct kvm_pmc *pmc)
{
struct kvm_pmu *pmu = pmc_to_pmu(pmc);
bool emulate_overflow;
u8 fixed_ctr_ctrl;
+ if (kvm_vcpu_has_mediated_pmu(pmu_to_vcpu(pmu))) {
+ kvm_mediated_pmu_refresh_event_filter(pmc);
+ return 0;
+ }
+
emulate_overflow = pmc_pause_counter(pmc);
if (!pmc_is_globally_enabled(pmc) || !pmc_is_locally_enabled(pmc) ||
return 0;
}
+ static bool kvm_need_any_pmc_intercept(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+
+ if (!kvm_vcpu_has_mediated_pmu(vcpu))
+ return true;
+
+ /*
+ * Note! Check *host* PMU capabilities, not KVM's PMU capabilities, as
+ * KVM's capabilities are constrained based on KVM support, i.e. KVM's
+ * capabilities themselves may be a subset of hardware capabilities.
+ */
+ return pmu->nr_arch_gp_counters != kvm_host_pmu.num_counters_gp ||
+ pmu->nr_arch_fixed_counters != kvm_host_pmu.num_counters_fixed;
+ }
+
+ bool kvm_need_perf_global_ctrl_intercept(struct kvm_vcpu *vcpu)
+ {
+ return kvm_need_any_pmc_intercept(vcpu) ||
+ !kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu));
+ }
+ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_perf_global_ctrl_intercept);
+
+ bool kvm_need_rdpmc_intercept(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+
+ /*
+ * VMware allows access to these Pseduo-PMCs even when read via RDPMC
+ * in Ring3 when CR4.PCE=0.
+ */
+ if (enable_vmware_backdoor)
+ return true;
+
+ return kvm_need_any_pmc_intercept(vcpu) ||
+ pmu->counter_bitmask[KVM_PMC_GP] != (BIT_ULL(kvm_host_pmu.bit_width_gp) - 1) ||
+ pmu->counter_bitmask[KVM_PMC_FIXED] != (BIT_ULL(kvm_host_pmu.bit_width_fixed) - 1);
+ }
+ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_rdpmc_intercept);
+
void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
{
if (lapic_in_kernel(vcpu)) {
pmu->global_ctrl = data;
reprogram_counters(pmu, diff);
}
+ /*
+ * Unconditionally forward writes to vendor code, i.e. to the
+ * VMC{B,S}, as pmu->global_ctrl is per-VCPU, not per-VMC{B,S}.
+ */
+ if (kvm_vcpu_has_mediated_pmu(vcpu))
+ kvm_pmu_call(write_global_ctrl)(data);
break;
case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
/*
pmc->counter = 0;
pmc->emulated_counter = 0;
- if (pmc_is_gp(pmc))
+ if (pmc_is_gp(pmc)) {
pmc->eventsel = 0;
+ pmc->eventsel_hw = 0;
+ }
}
- pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0;
+ pmu->fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw = 0;
+ pmu->global_ctrl = pmu->global_status = 0;
kvm_pmu_call(reset)(vcpu);
}
{
struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
- if (KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm))
+ if (KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm))
return;
/*
* in the global controls). Emulate that behavior when refreshing the
* PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL.
*/
- if (kvm_pmu_has_perf_global_ctrl(pmu) && pmu->nr_arch_gp_counters)
+ if (pmu->nr_arch_gp_counters &&
+ (kvm_pmu_has_perf_global_ctrl(pmu) || kvm_vcpu_has_mediated_pmu(vcpu)))
pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0);
+ if (kvm_vcpu_has_mediated_pmu(vcpu))
+ kvm_pmu_call(write_global_ctrl)(pmu->global_ctrl);
+
bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters);
bitmap_set(pmu->all_valid_pmc_idx, KVM_FIXED_PMC_BASE_IDX,
pmu->nr_arch_fixed_counters);
kvm_pmu_reset(vcpu);
}
+ static bool pmc_is_pmi_enabled(struct kvm_pmc *pmc)
+ {
+ u8 fixed_ctr_ctrl;
+
+ if (pmc_is_gp(pmc))
+ return pmc->eventsel & ARCH_PERFMON_EVENTSEL_INT;
+
+ fixed_ctr_ctrl = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl,
+ pmc->idx - KVM_FIXED_PMC_BASE_IDX);
+ return fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI;
+ }
+
static void kvm_pmu_incr_counter(struct kvm_pmc *pmc)
{
- pmc->emulated_counter++;
- kvm_pmu_request_counter_reprogram(pmc);
+ struct kvm_vcpu *vcpu = pmc->vcpu;
+
+ /*
+ * For perf-based PMUs, accumulate software-emulated events separately
+ * from pmc->counter, as pmc->counter is offset by the count of the
+ * associated perf event. Request reprogramming, which will consult
+ * both emulated and hardware-generated events to detect overflow.
+ */
+ if (!kvm_vcpu_has_mediated_pmu(vcpu)) {
+ pmc->emulated_counter++;
+ kvm_pmu_request_counter_reprogram(pmc);
+ return;
+ }
+
+ /*
+ * For mediated PMUs, pmc->counter is updated when the vCPU's PMU is
+ * put, and will be loaded into hardware when the PMU is loaded. Simply
+ * increment the counter and signal overflow if it wraps to zero.
+ */
+ pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc);
+ if (!pmc->counter) {
+ pmc_to_pmu(pmc)->global_status |= BIT_ULL(pmc->idx);
+ if (pmc_is_pmi_enabled(pmc))
+ kvm_make_request(KVM_REQ_PMI, vcpu);
+ }
}
static inline bool cpl_is_matched(struct kvm_pmc *pmc)
kfree(filter);
return r;
}
+
+ static __always_inline u32 fixed_counter_msr(u32 idx)
+ {
+ return kvm_pmu_ops.FIXED_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
+ }
+
+ static __always_inline u32 gp_counter_msr(u32 idx)
+ {
+ return kvm_pmu_ops.GP_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
+ }
+
+ static __always_inline u32 gp_eventsel_msr(u32 idx)
+ {
+ return kvm_pmu_ops.GP_EVENTSEL_BASE + idx * kvm_pmu_ops.MSR_STRIDE;
+ }
+
+ static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+ struct kvm_pmc *pmc;
+ u32 i;
+
+ /*
+ * No need to zero out unexposed GP/fixed counters/selectors since RDPMC
+ * is intercepted if hardware has counters that aren't visible to the
+ * guest (KVM will inject #GP as appropriate).
+ */
+ for (i = 0; i < pmu->nr_arch_gp_counters; i++) {
+ pmc = &pmu->gp_counters[i];
+
+ if (pmc->counter != rdpmc(i))
+ wrmsrl(gp_counter_msr(i), pmc->counter);
+ wrmsrl(gp_eventsel_msr(i), pmc->eventsel_hw);
+ }
+ for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
+ pmc = &pmu->fixed_counters[i];
+
+ if (pmc->counter != rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i))
+ wrmsrl(fixed_counter_msr(i), pmc->counter);
+ }
+ }
+
+ void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu)
+ {
+ if (!kvm_vcpu_has_mediated_pmu(vcpu) ||
+ KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
+ return;
+
+ lockdep_assert_irqs_disabled();
+
+ perf_load_guest_context();
+
+ /*
+ * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context
+ * disables all individual counters (if any were enabled), but doesn't
+ * globally disable the entire PMU. Loading event selectors and PMCs
+ * with guest values while PERF_GLOBAL_CTRL is non-zero will generate
+ * unexpected events and PMIs.
+ *
+ * VMX will enable/disable counters at VM-Enter/VM-Exit by atomically
+ * loading PERF_GLOBAL_CONTROL. SVM effectively performs the switch by
+ * configuring all events to be GUEST_ONLY. Clear PERF_GLOBAL_CONTROL
+ * even for SVM to minimize the damage if a perf event is left enabled,
+ * and to ensure a consistent starting state.
+ */
+ wrmsrq(kvm_pmu_ops.PERF_GLOBAL_CTRL, 0);
+
+ perf_load_guest_lvtpc(kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVTPC));
+
+ kvm_pmu_load_guest_pmcs(vcpu);
+
+ kvm_pmu_call(mediated_load)(vcpu);
+ }
+
+ static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+ struct kvm_pmc *pmc;
+ u32 i;
+
+ /*
+ * Clear selectors and counters to ensure hardware doesn't count using
+ * guest controls when the host (perf) restores its state.
+ */
+ for (i = 0; i < pmu->nr_arch_gp_counters; i++) {
+ pmc = &pmu->gp_counters[i];
+
+ pmc->counter = rdpmc(i);
+ if (pmc->counter)
+ wrmsrq(gp_counter_msr(i), 0);
+ if (pmc->eventsel_hw)
+ wrmsrq(gp_eventsel_msr(i), 0);
+ }
+
+ for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
+ pmc = &pmu->fixed_counters[i];
+
+ pmc->counter = rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i);
+ if (pmc->counter)
+ wrmsrq(fixed_counter_msr(i), 0);
+ }
+ }
+
+ void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu)
+ {
+ if (!kvm_vcpu_has_mediated_pmu(vcpu) ||
+ KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm))
+ return;
+
+ lockdep_assert_irqs_disabled();
+
+ /*
+ * Defer handling of PERF_GLOBAL_CTRL to vendor code. On Intel, it's
+ * atomically cleared on VM-Exit, i.e. doesn't need to be clear here.
+ */
+ kvm_pmu_call(mediated_put)(vcpu);
+
+ kvm_pmu_put_guest_pmcs(vcpu);
+
+ perf_put_guest_lvtpc();
+
+ perf_put_guest_context();
+ }
* correctly fill in the high bits of exit_info_1.
*/
vmcb->control.exit_code = SVM_EXIT_NPF;
- vmcb->control.exit_code_hi = 0;
vmcb->control.exit_info_1 = (1ULL << 32);
vmcb->control.exit_info_2 = fault->address;
}
* Hardcode the capacity of the array based on the maximum number of _offsets_.
* MSRs are batched together, so there are fewer offsets than MSRs.
*/
- static int nested_svm_msrpm_merge_offsets[7] __ro_after_init;
+ static int nested_svm_msrpm_merge_offsets[10] __ro_after_init;
static int nested_svm_nr_msrpm_merge_offsets __ro_after_init;
typedef unsigned long nsvm_msrpm_merge_t;
MSR_IA32_LASTBRANCHTOIP,
MSR_IA32_LASTINTFROMIP,
MSR_IA32_LASTINTTOIP,
+
+ MSR_K7_PERFCTR0,
+ MSR_K7_PERFCTR1,
+ MSR_K7_PERFCTR2,
+ MSR_K7_PERFCTR3,
+ MSR_F15H_PERF_CTR0,
+ MSR_F15H_PERF_CTR1,
+ MSR_F15H_PERF_CTR2,
+ MSR_F15H_PERF_CTR3,
+ MSR_F15H_PERF_CTR4,
+ MSR_F15H_PERF_CTR5,
+
+ MSR_AMD64_PERF_CNTR_GLOBAL_CTL,
+ MSR_AMD64_PERF_CNTR_GLOBAL_STATUS,
+ MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
+ MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET,
};
int i, j;
return __nested_vmcb_check_controls(vcpu, ctl);
}
+/*
+ * If a feature is not advertised to L1, clear the corresponding vmcb12
+ * intercept.
+ */
+#define __nested_svm_sanitize_intercept(__vcpu, __control, fname, iname) \
+do { \
+ if (!guest_cpu_cap_has(__vcpu, X86_FEATURE_##fname)) \
+ vmcb12_clr_intercept(__control, INTERCEPT_##iname); \
+} while (0)
+
+#define nested_svm_sanitize_intercept(__vcpu, __control, name) \
+ __nested_svm_sanitize_intercept(__vcpu, __control, name, name)
+
static
void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu,
struct vmcb_ctrl_area_cached *to,
for (i = 0; i < MAX_INTERCEPT; i++)
to->intercepts[i] = from->intercepts[i];
+ __nested_svm_sanitize_intercept(vcpu, to, XSAVE, XSETBV);
+ nested_svm_sanitize_intercept(vcpu, to, INVPCID);
+ nested_svm_sanitize_intercept(vcpu, to, RDTSCP);
+ nested_svm_sanitize_intercept(vcpu, to, SKINIT);
+ nested_svm_sanitize_intercept(vcpu, to, RDPRU);
+
to->iopm_base_pa = from->iopm_base_pa;
to->msrpm_base_pa = from->msrpm_base_pa;
to->tsc_offset = from->tsc_offset;
to->tlb_ctl = from->tlb_ctl;
+ to->erap_ctl = from->erap_ctl;
to->int_ctl = from->int_ctl;
to->int_vector = from->int_vector;
to->int_state = from->int_state;
to->exit_code = from->exit_code;
- to->exit_code_hi = from->exit_code_hi;
to->exit_info_1 = from->exit_info_1;
to->exit_info_2 = from->exit_info_2;
to->exit_int_info = from->exit_int_info;
vmcb02->save.rsp = vmcb12->save.rsp;
vmcb02->save.rip = vmcb12->save.rip;
- /* These bits will be set properly on the first execution when new_vmc12 is true */
if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) {
vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1;
svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW;
enter_guest_mode(vcpu);
/*
- * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2,
- * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes.
+ * Filled at exit: exit_code, exit_info_1, exit_info_2, exit_int_info,
+ * exit_int_info_err, next_rip, insn_len, insn_bytes.
*/
if (guest_cpu_cap_has(vcpu, X86_FEATURE_VGIF) &&
}
}
+ /*
+ * Take ALLOW_LARGER_RAP from vmcb12 even though it should be safe to
+ * let L2 use a larger RAP since KVM will emulate the necessary clears,
+ * as it's possible L1 deliberately wants to restrict L2 to the legacy
+ * RAP size. Unconditionally clear the RAP on nested VMRUN, as KVM is
+ * responsible for emulating the host vs. guest tags (L1 is the "host",
+ * L2 is the "guest").
+ */
+ if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
+ vmcb02->control.erap_ctl = (svm->nested.ctl.erap_ctl &
+ ERAP_CONTROL_ALLOW_LARGER_RAP) |
+ ERAP_CONTROL_CLEAR_RAP;
+
/*
* Merge guest and host intercepts - must be called with vcpu in
* guest-mode to take effect.
if (!nested_vmcb_check_save(vcpu) ||
!nested_vmcb_check_controls(vcpu)) {
vmcb12->control.exit_code = SVM_EXIT_ERR;
- vmcb12->control.exit_code_hi = 0;
vmcb12->control.exit_info_1 = 0;
vmcb12->control.exit_info_2 = 0;
goto out;
svm->soft_int_injected = false;
svm->vmcb->control.exit_code = SVM_EXIT_ERR;
- svm->vmcb->control.exit_code_hi = 0;
svm->vmcb->control.exit_info_1 = 0;
svm->vmcb->control.exit_info_2 = 0;
vmcb12->control.int_state = vmcb02->control.int_state;
vmcb12->control.exit_code = vmcb02->control.exit_code;
- vmcb12->control.exit_code_hi = vmcb02->control.exit_code_hi;
vmcb12->control.exit_info_1 = vmcb02->control.exit_info_1;
vmcb12->control.exit_info_2 = vmcb02->control.exit_info_2;
- if (vmcb12->control.exit_code != SVM_EXIT_ERR)
+ if (!svm_is_vmrun_failure(vmcb12->control.exit_code))
nested_save_pending_event_to_vmcb12(svm, vmcb12);
if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
kvm_nested_vmexit_handle_ibrs(vcpu);
+ if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
+ vmcb01->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP;
+
svm_switch_vmcb(svm, &svm->vmcb01);
/*
nested_svm_uninit_mmu_context(vcpu);
vmcb_mark_all_dirty(svm->vmcb);
+ svm_set_gif(svm, true);
+
if (kvm_apicv_activated(vcpu->kvm))
kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
}
static int nested_svm_intercept(struct vcpu_svm *svm)
{
- u32 exit_code = svm->vmcb->control.exit_code;
+ u64 exit_code = svm->vmcb->control.exit_code;
int vmexit = NESTED_EXIT_HOST;
+ if (svm_is_vmrun_failure(exit_code))
+ return NESTED_EXIT_DONE;
+
switch (exit_code) {
case SVM_EXIT_MSR:
vmexit = nested_svm_exit_handled_msr(svm);
case SVM_EXIT_IOIO:
vmexit = nested_svm_intercept_ioio(svm);
break;
- case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
+ case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f:
/*
* Host-intercepted exceptions have been checked already in
* nested_svm_exit_special. There is nothing to do here,
*/
vmexit = NESTED_EXIT_DONE;
break;
- }
- case SVM_EXIT_ERR: {
- vmexit = NESTED_EXIT_DONE;
- break;
- }
- default: {
+ default:
if (vmcb12_is_intercept(&svm->nested.ctl, exit_code))
vmexit = NESTED_EXIT_DONE;
- }
+ break;
}
return vmexit;
struct vmcb *vmcb = svm->vmcb;
vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + ex->vector;
- vmcb->control.exit_code_hi = 0;
if (ex->has_error_code)
vmcb->control.exit_info_1 = ex->error_code;
dst->tsc_offset = from->tsc_offset;
dst->asid = from->asid;
dst->tlb_ctl = from->tlb_ctl;
+ dst->erap_ctl = from->erap_ctl;
dst->int_ctl = from->int_ctl;
dst->int_vector = from->int_vector;
dst->int_state = from->int_state;
dst->exit_code = from->exit_code;
- dst->exit_code_hi = from->exit_code_hi;
dst->exit_info_1 = from->exit_info_1;
dst->exit_info_2 = from->exit_info_2;
dst->exit_int_info = from->exit_int_info;
/*
* If in guest mode, vcpu->arch.efer actually refers to the L2 guest's
* EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed.
+ * If SVME is disabled, the only valid states are "none" and GIF=1
+ * (clearing SVME does NOT set GIF, i.e. GIF=0 is allowed).
*/
- if (!(vcpu->arch.efer & EFER_SVME)) {
- /* GIF=1 and no guest mode are required if SVME=0. */
- if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET)
- return -EINVAL;
- }
+ if (!(vcpu->arch.efer & EFER_SVME) && kvm_state->flags &&
+ kvm_state->flags != KVM_STATE_NESTED_GIF_SET)
+ return -EINVAL;
/* SMM temporarily disables SVM, so we cannot be in guest mode. */
if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
* thus MMU might not be initialized correctly.
* Set it again to fix this.
*/
-
ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3,
nested_npt_enabled(svm), false);
- if (WARN_ON_ONCE(ret))
+ if (ret)
goto out_free;
svm->nested.force_msr_bitmap_recalc = true;
bool vnmi = true;
module_param(vnmi, bool, 0444);
+ module_param(enable_mediated_pmu, bool, 0444);
+
static bool svm_gp_erratum_intercept = true;
static u8 rsm_ins_bytes[] = "\x0f\xaa";
if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
if (!(efer & EFER_SVME)) {
svm_leave_nested(vcpu);
- svm_set_gif(svm, true);
/* #GP intercept is still needed for vmware backdoor */
if (!enable_vmware_backdoor)
clr_exception_intercept(svm, GP_VECTOR);
__free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
}
+ static void svm_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
+ {
+ bool intercept = !kvm_vcpu_has_mediated_pmu(vcpu);
+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+ int i;
+
+ if (!enable_mediated_pmu)
+ return;
+
+ /* Legacy counters are always available for AMD CPUs with a PMU. */
+ for (i = 0; i < min(pmu->nr_arch_gp_counters, AMD64_NUM_COUNTERS); i++)
+ svm_set_intercept_for_msr(vcpu, MSR_K7_PERFCTR0 + i,
+ MSR_TYPE_RW, intercept);
+
+ intercept |= !guest_cpu_cap_has(vcpu, X86_FEATURE_PERFCTR_CORE);
+ for (i = 0; i < pmu->nr_arch_gp_counters; i++)
+ svm_set_intercept_for_msr(vcpu, MSR_F15H_PERF_CTR + 2 * i,
+ MSR_TYPE_RW, intercept);
+
+ for ( ; i < kvm_pmu_cap.num_counters_gp; i++)
+ svm_enable_intercept_for_msr(vcpu, MSR_F15H_PERF_CTR + 2 * i,
+ MSR_TYPE_RW);
+
+ intercept = kvm_need_perf_global_ctrl_intercept(vcpu);
+ svm_set_intercept_for_msr(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_CTL,
+ MSR_TYPE_RW, intercept);
+ svm_set_intercept_for_msr(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_STATUS,
+ MSR_TYPE_RW, intercept);
+ svm_set_intercept_for_msr(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
+ MSR_TYPE_RW, intercept);
+ svm_set_intercept_for_msr(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET,
+ MSR_TYPE_RW, intercept);
+ }
+
static void svm_recalc_msr_intercepts(struct kvm_vcpu *vcpu)
{
struct vcpu_svm *svm = to_svm(vcpu);
if (sev_es_guest(vcpu->kvm))
sev_es_recalc_msr_intercepts(vcpu);
+ svm_recalc_pmu_msr_intercepts(vcpu);
+
/*
* x2APIC intercepts are modified on-demand and cannot be filtered by
* userspace.
svm_set_intercept(svm, INTERCEPT_RDTSCP);
}
+ /*
+ * No need to toggle VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK here, it is
+ * always set if vls is enabled. If the intercepts are set, the bit is
+ * meaningless anyway.
+ */
if (guest_cpuid_is_intel_compatible(vcpu)) {
svm_set_intercept(svm, INTERCEPT_VMLOAD);
svm_set_intercept(svm, INTERCEPT_VMSAVE);
- svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
} else {
/*
* If hardware supports Virtual VMLOAD VMSAVE then enable it
if (vls) {
svm_clr_intercept(svm, INTERCEPT_VMLOAD);
svm_clr_intercept(svm, INTERCEPT_VMSAVE);
- svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
}
}
+
+ if (kvm_need_rdpmc_intercept(vcpu))
+ svm_set_intercept(svm, INTERCEPT_RDPMC);
+ else
+ svm_clr_intercept(svm, INTERCEPT_RDPMC);
}
static void svm_recalc_intercepts(struct kvm_vcpu *vcpu)
svm_clr_intercept(svm, INTERCEPT_PAUSE);
}
+ if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
+ svm->vmcb->control.erap_ctl |= ERAP_CONTROL_ALLOW_LARGER_RAP;
+
if (kvm_vcpu_apicv_active(vcpu))
avic_init_vmcb(svm, vmcb);
svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
}
+ if (vls)
+ svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
+
if (vcpu->kvm->arch.bus_lock_detection_enabled)
svm_set_intercept(svm, INTERCEPT_BUSLOCK);
svm->vmcb->control.insn_len);
}
+static int svm_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
+ void *insn, int insn_len);
+
static int npf_interception(struct kvm_vcpu *vcpu)
{
struct vcpu_svm *svm = to_svm(vcpu);
int rc;
- u64 fault_address = svm->vmcb->control.exit_info_2;
u64 error_code = svm->vmcb->control.exit_info_1;
+ gpa_t gpa = svm->vmcb->control.exit_info_2;
/*
* WARN if hardware generates a fault with an error code that collides
if (WARN_ON_ONCE(error_code & PFERR_SYNTHETIC_MASK))
error_code &= ~PFERR_SYNTHETIC_MASK;
+ /*
+ * Expedite fast MMIO kicks if the next RIP is known and KVM is allowed
+ * emulate a page fault, e.g. skipping the current instruction is wrong
+ * if the #NPF occurred while vectoring an event.
+ */
+ if ((error_code & PFERR_RSVD_MASK) && !is_guest_mode(vcpu)) {
+ const int emul_type = EMULTYPE_PF | EMULTYPE_NO_DECODE;
+
+ if (svm_check_emulate_instruction(vcpu, emul_type, NULL, 0))
+ return 1;
+
+ if (nrips && svm->vmcb->control.next_rip &&
+ !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
+ trace_kvm_fast_mmio(gpa);
+ return kvm_skip_emulated_instruction(vcpu);
+ }
+ }
+
if (sev_snp_guest(vcpu->kvm) && (error_code & PFERR_GUEST_ENC_MASK))
error_code |= PFERR_PRIVATE_ACCESS;
- trace_kvm_page_fault(vcpu, fault_address, error_code);
- rc = kvm_mmu_page_fault(vcpu, fault_address, error_code,
+ trace_kvm_page_fault(vcpu, gpa, error_code);
+ rc = kvm_mmu_page_fault(vcpu, gpa, error_code,
static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
svm->vmcb->control.insn_bytes : NULL,
svm->vmcb->control.insn_len);
if (rc > 0 && error_code & PFERR_GUEST_RMP_MASK)
- sev_handle_rmp_fault(vcpu, fault_address, error_code);
+ sev_handle_rmp_fault(vcpu, gpa, error_code);
return rc;
}
ret = kvm_skip_emulated_instruction(vcpu);
+ /* KVM always performs VMLOAD/VMSAVE on VMCB01 (see __svm_vcpu_run()) */
if (vmload) {
- svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
+ svm_copy_vmloadsave_state(svm->vmcb01.ptr, vmcb12);
svm->sysenter_eip_hi = 0;
svm->sysenter_esp_hi = 0;
} else {
- svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
+ svm_copy_vmloadsave_state(vmcb12, svm->vmcb01.ptr);
}
kvm_vcpu_unmap(vcpu, &map);
pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
pr_err("%-20s%d\n", "asid:", control->asid);
pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
+ pr_err("%-20s%d\n", "erap_ctl:", control->erap_ctl);
pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
pr_err("%-20s%08x\n", "int_state:", control->int_state);
- pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
+ pr_err("%-20s%016llx\n", "exit_code:", control->exit_code);
pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
sev_free_decrypted_vmsa(vcpu, save);
}
-static bool svm_check_exit_valid(u64 exit_code)
+int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 __exit_code)
{
- return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
- svm_exit_handlers[exit_code]);
-}
-
-static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
-{
- dump_vmcb(vcpu);
- kvm_prepare_unexpected_reason_exit(vcpu, exit_code);
- return 0;
-}
+ u32 exit_code = __exit_code;
-int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
-{
- if (!svm_check_exit_valid(exit_code))
- return svm_handle_invalid_exit(vcpu, exit_code);
+ /*
+ * SVM uses negative values, i.e. 64-bit values, to indicate that VMRUN
+ * failed. Report all such errors to userspace (note, VMEXIT_INVALID,
+ * a.k.a. SVM_EXIT_ERR, is special cased by svm_handle_exit()). Skip
+ * the check when running as a VM, as KVM has historically left garbage
+ * in bits 63:32, i.e. running KVM-on-KVM would hit false positives if
+ * the underlying kernel is buggy.
+ */
+ if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR) &&
+ (u64)exit_code != __exit_code)
+ goto unexpected_vmexit;
#ifdef CONFIG_MITIGATION_RETPOLINE
if (exit_code == SVM_EXIT_MSR)
return sev_handle_vmgexit(vcpu);
#endif
#endif
+ if (exit_code >= ARRAY_SIZE(svm_exit_handlers))
+ goto unexpected_vmexit;
+
+ exit_code = array_index_nospec(exit_code, ARRAY_SIZE(svm_exit_handlers));
+ if (!svm_exit_handlers[exit_code])
+ goto unexpected_vmexit;
+
return svm_exit_handlers[exit_code](vcpu);
+
+unexpected_vmexit:
+ dump_vmcb(vcpu);
+ kvm_prepare_unexpected_reason_exit(vcpu, __exit_code);
+ return 0;
}
static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
{
struct vcpu_svm *svm = to_svm(vcpu);
struct kvm_run *kvm_run = vcpu->run;
- u32 exit_code = svm->vmcb->control.exit_code;
/* SEV-ES guests must use the CR write traps to track CR registers. */
if (!sev_es_guest(vcpu->kvm)) {
return 1;
}
- if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
+ if (svm_is_vmrun_failure(svm->vmcb->control.exit_code)) {
kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
kvm_run->fail_entry.hardware_entry_failure_reason
= svm->vmcb->control.exit_code;
if (exit_fastpath != EXIT_FASTPATH_NONE)
return 1;
- return svm_invoke_exit_handler(vcpu, exit_code);
+ return svm_invoke_exit_handler(vcpu, svm->vmcb->control.exit_code);
}
static int pre_svm_run(struct kvm_vcpu *vcpu)
invlpga(gva, svm->vmcb->control.asid);
}
+static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu)
+{
+ kvm_register_mark_dirty(vcpu, VCPU_EXREG_ERAPS);
+
+ svm_flush_tlb_asid(vcpu);
+}
+
static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
{
struct vcpu_svm *svm = to_svm(vcpu);
}
svm->vmcb->save.cr2 = vcpu->arch.cr2;
+ if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS) &&
+ kvm_register_is_dirty(vcpu, VCPU_EXREG_ERAPS))
+ svm->vmcb->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP;
+
svm_hv_update_vp_id(svm->vmcb, vcpu);
/*
/* Track VMRUNs that have made past consistency checking */
if (svm->nested.nested_run_pending &&
- svm->vmcb->control.exit_code != SVM_EXIT_ERR)
+ !svm_is_vmrun_failure(svm->vmcb->control.exit_code))
++vcpu->stat.nested_run;
svm->nested.nested_run_pending = 0;
}
svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
+
+ /*
+ * Unconditionally mask off the CLEAR_RAP bit, the AND is just as cheap
+ * as the TEST+Jcc to avoid it.
+ */
+ if (cpu_feature_enabled(X86_FEATURE_ERAPS))
+ svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP;
+
vmcb_mark_all_clean(svm->vmcb);
/* if exit due to PF check for async PF */
vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
+ if (!msr_write_intercepted(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_CTL))
+ rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, vcpu_to_pmu(vcpu)->global_ctrl);
+
trace_kvm_exit(vcpu, KVM_ISA_SVM);
svm_complete_interrupts(vcpu);
.flush_tlb_all = svm_flush_tlb_all,
.flush_tlb_current = svm_flush_tlb_current,
.flush_tlb_gva = svm_flush_tlb_gva,
- .flush_tlb_guest = svm_flush_tlb_asid,
+ .flush_tlb_guest = svm_flush_tlb_guest,
.vcpu_pre_run = svm_vcpu_pre_run,
.vcpu_run = svm_vcpu_run,
static __init void svm_set_cpu_caps(void)
{
- kvm_set_cpu_caps();
+ kvm_initialize_cpu_caps();
kvm_caps.supported_perf_cap = 0;
*/
kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT);
kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM);
+
+ kvm_setup_xss_caps();
+ kvm_finalize_cpu_caps();
}
static __init int svm_hardware_setup(void)
#include "trace.h"
#include "vmx.h"
#include "smm.h"
+#include "x86_ops.h"
static bool __read_mostly enable_shadow_vmcs = 1;
module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
pr_err("Missing field from shadow_read_only_field %x\n",
field + 1);
+ if (get_vmcs12_field_offset(field) < 0)
+ continue;
+
clear_bit(field, vmx_vmread_bitmap);
if (field & 1)
#ifdef CONFIG_X86_64
field <= GUEST_TR_AR_BYTES,
"Update vmcs12_write_any() to drop reserved bits from AR_BYTES");
+ if (get_vmcs12_field_offset(field) < 0)
+ continue;
+
/*
- * PML and the preemption timer can be emulated, but the
- * processor cannot vmwrite to fields that don't exist
- * on bare metal.
+ * KVM emulates PML and the VMX preemption timer irrespective
+ * of hardware support, but shadowing their related VMCS fields
+ * requires hardware support as the CPU will reject VMWRITEs to
+ * fields that don't exist.
*/
switch (field) {
case GUEST_PML_INDEX:
if (!cpu_has_vmx_preemption_timer())
continue;
break;
- case GUEST_INTR_STATUS:
- if (!cpu_has_vmx_apicv())
- continue;
- break;
default:
break;
}
msr_bitmap_l0, msr);
}
+ #define nested_vmx_merge_msr_bitmaps(msr, type) \
+ nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, \
+ msr_bitmap_l0, msr, type)
+
+ #define nested_vmx_merge_msr_bitmaps_read(msr) \
+ nested_vmx_merge_msr_bitmaps(msr, MSR_TYPE_R)
+
+ #define nested_vmx_merge_msr_bitmaps_write(msr) \
+ nested_vmx_merge_msr_bitmaps(msr, MSR_TYPE_W)
+
+ #define nested_vmx_merge_msr_bitmaps_rw(msr) \
+ nested_vmx_merge_msr_bitmaps(msr, MSR_TYPE_RW)
+
+ static void nested_vmx_merge_pmu_msr_bitmaps(struct kvm_vcpu *vcpu,
+ unsigned long *msr_bitmap_l1,
+ unsigned long *msr_bitmap_l0)
+ {
+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+ int i;
+
+ /*
+ * Skip the merges if the vCPU doesn't have a mediated PMU MSR, i.e. if
+ * none of the MSRs can possibly be passed through to L1.
+ */
+ if (!kvm_vcpu_has_mediated_pmu(vcpu))
+ return;
+
+ for (i = 0; i < pmu->nr_arch_gp_counters; i++) {
+ nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_PERFCTR0 + i);
+ nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_PMC0 + i);
+ }
+
+ for (i = 0; i < pmu->nr_arch_fixed_counters; i++)
+ nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_FIXED_CTR0 + i);
+
+ nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_GLOBAL_CTRL);
+ nested_vmx_merge_msr_bitmaps_read(MSR_CORE_PERF_GLOBAL_STATUS);
+ nested_vmx_merge_msr_bitmaps_write(MSR_CORE_PERF_GLOBAL_OVF_CTRL);
+ }
+
/*
* Merge L0's and L1's MSR bitmap, return false to indicate that
* we do not use the hardware.
* other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through.
*/
#ifdef CONFIG_X86_64
- nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
- MSR_FS_BASE, MSR_TYPE_RW);
-
- nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
- MSR_GS_BASE, MSR_TYPE_RW);
-
- nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
- MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
+ nested_vmx_merge_msr_bitmaps_rw(MSR_FS_BASE);
+ nested_vmx_merge_msr_bitmaps_rw(MSR_GS_BASE);
+ nested_vmx_merge_msr_bitmaps_rw(MSR_KERNEL_GS_BASE);
#endif
- nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
- MSR_IA32_SPEC_CTRL, MSR_TYPE_RW);
-
- nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
- MSR_IA32_PRED_CMD, MSR_TYPE_W);
-
- nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
- MSR_IA32_FLUSH_CMD, MSR_TYPE_W);
+ nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_SPEC_CTRL);
+ nested_vmx_merge_msr_bitmaps_write(MSR_IA32_PRED_CMD);
+ nested_vmx_merge_msr_bitmaps_write(MSR_IA32_FLUSH_CMD);
nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
MSR_IA32_APERF, MSR_TYPE_R);
nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0,
MSR_IA32_PL3_SSP, MSR_TYPE_RW);
+ nested_vmx_merge_pmu_msr_bitmaps(vcpu, msr_bitmap_l1, msr_bitmap_l0);
+
kvm_vcpu_unmap(vcpu, &map);
vmx->nested.force_msr_bitmap_recalc = false;
* does not include the time taken for emulation of the L2->L1
* VM-exit in L0, use the more accurate value.
*/
- if (msr_index == MSR_IA32_TSC) {
- int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest,
- MSR_IA32_TSC);
-
- if (i >= 0) {
- u64 val = vmx->msr_autostore.guest.val[i].value;
+ if (msr_index == MSR_IA32_TSC && vmx->nested.tsc_autostore_slot >= 0) {
+ int slot = vmx->nested.tsc_autostore_slot;
+ u64 host_tsc = vmx->msr_autostore.val[slot].value;
- *data = kvm_read_l1_tsc(vcpu, val);
- return true;
- }
+ *data = kvm_read_l1_tsc(vcpu, host_tsc);
+ return true;
}
if (kvm_emulate_msr_read(vcpu, msr_index, data)) {
return false;
}
- static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu,
- u32 msr_index)
- {
- struct vcpu_vmx *vmx = to_vmx(vcpu);
- struct vmx_msrs *autostore = &vmx->msr_autostore.guest;
- bool in_vmcs12_store_list;
- int msr_autostore_slot;
- bool in_autostore_list;
- int last;
-
- msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index);
- in_autostore_list = msr_autostore_slot >= 0;
- in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index);
-
- if (in_vmcs12_store_list && !in_autostore_list) {
- if (autostore->nr == MAX_NR_LOADSTORE_MSRS) {
- /*
- * Emulated VMEntry does not fail here. Instead a less
- * accurate value will be returned by
- * nested_vmx_get_vmexit_msr_value() by reading KVM's
- * internal MSR state instead of reading the value from
- * the vmcs02 VMExit MSR-store area.
- */
- pr_warn_ratelimited(
- "Not enough msr entries in msr_autostore. Can't add msr %x\n",
- msr_index);
- return;
- }
- last = autostore->nr++;
- autostore->val[last].index = msr_index;
- } else if (!in_vmcs12_store_list && in_autostore_list) {
- last = --autostore->nr;
- autostore->val[msr_autostore_slot] = autostore->val[last];
- }
- }
-
/*
* Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are
* emulating VM-Entry into a guest with EPT enabled. On failure, the expected
* addresses are constant (for vmcs02), the counts can change based
* on L2's behavior, e.g. switching to/from long mode.
*/
- vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val));
+ vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.val));
vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
exec_control &= ~CPU_BASED_TPR_SHADOW;
exec_control |= vmcs12->cpu_based_vm_exec_control;
- vmx->nested.l1_tpr_threshold = -1;
if (exec_control & CPU_BASED_TPR_SHADOW)
vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
#ifdef CONFIG_X86_64
}
/*
- * Make sure the msr_autostore list is up to date before we set the
- * count in the vmcs02.
+ * If vmcs12 is configured to save TSC on exit via the auto-store list,
+ * append the MSR to vmcs02's auto-store list so that KVM effectively
+ * reads TSC at the time of VM-Exit from L2. The saved value will be
+ * propagated to vmcs12's list on nested VM-Exit.
+ *
+ * Don't increment the number of MSRs in the vCPU structure, as saving
+ * TSC is specific to this particular incarnation of vmcb02, i.e. must
+ * not bleed into vmcs01.
*/
- prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC);
+ if (nested_msr_store_list_has_msr(&vmx->vcpu, MSR_IA32_TSC) &&
+ !WARN_ON_ONCE(vmx->msr_autostore.nr >= ARRAY_SIZE(vmx->msr_autostore.val))) {
+ vmx->nested.tsc_autostore_slot = vmx->msr_autostore.nr;
+ vmx->msr_autostore.val[vmx->msr_autostore.nr].index = MSR_IA32_TSC;
- vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr);
+ vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.nr + 1);
+ } else {
+ vmx->nested.tsc_autostore_slot = -1;
+ vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.nr);
+ }
vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
}
}
-
-void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
-{
- struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
- gfn_t gfn;
-
- /*
- * Don't need to mark the APIC access page dirty; it is never
- * written to by the CPU during APIC virtualization.
- */
-
- if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
- gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
- kvm_vcpu_mark_page_dirty(vcpu, gfn);
- }
-
- if (nested_cpu_has_posted_intr(vmcs12)) {
- gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
- kvm_vcpu_mark_page_dirty(vcpu, gfn);
- }
-}
-
static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
}
}
- nested_mark_vmcs12_pages_dirty(vcpu);
+ kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.virtual_apic_map);
+ kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.pi_desc_map);
return 0;
mmio_needed:
kvm_nested_vmexit_handle_ibrs(vcpu);
- /* Update any VMCS fields that might have changed while L2 ran */
+ /*
+ * Update any VMCS fields that might have changed while vmcs02 was the
+ * active VMCS. The tracking is per-vCPU, not per-VMCS.
+ */
+ vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.nr);
vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
if (kvm_caps.has_tsc_control)
vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio);
- if (vmx->nested.l1_tpr_threshold != -1)
- vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold);
-
- if (vmx->nested.change_vmcs01_virtual_apic_mode) {
- vmx->nested.change_vmcs01_virtual_apic_mode = false;
- vmx_set_virtual_apic_mode(vcpu);
- }
-
- if (vmx->nested.update_vmcs01_cpu_dirty_logging) {
- vmx->nested.update_vmcs01_cpu_dirty_logging = false;
- vmx_update_cpu_dirty_logging(vcpu);
- }
-
nested_put_vmcs12_pages(vcpu);
- if (vmx->nested.reload_vmcs01_apic_access_page) {
- vmx->nested.reload_vmcs01_apic_access_page = false;
- kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
- }
-
- if (vmx->nested.update_vmcs01_apicv_status) {
- vmx->nested.update_vmcs01_apicv_status = false;
- kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
- }
-
- if (vmx->nested.update_vmcs01_hwapic_isr) {
- vmx->nested.update_vmcs01_hwapic_isr = false;
- kvm_apic_update_hwapic_isr(vcpu);
- }
-
if ((vm_exit_reason != -1) &&
(enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)))
vmx->nested.need_vmcs12_to_shadow_sync = true;
}
}
-/*
- * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6. Undo
- * that madness to get the encoding for comparison.
- */
-#define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10)))
-
static u64 nested_vmx_calc_vmcs_enum_msr(void)
{
/*
{
int i;
+ /*
+ * Note! The set of supported vmcs12 fields is consumed by both VMX
+ * MSR and shadow VMCS setup.
+ */
+ nested_vmx_setup_vmcs12_fields();
+
+ nested_vmx_setup_ctls_msrs(&vmcs_config, vmx_capability.ept);
+
if (!cpu_has_vmx_shadow_vmcs())
enable_shadow_vmcs = 0;
if (enable_shadow_vmcs) {
extern bool __read_mostly allow_smaller_maxphyaddr;
module_param(allow_smaller_maxphyaddr, bool, S_IRUGO);
+ module_param(enable_mediated_pmu, bool, 0444);
+
#define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD)
#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
#define KVM_VM_CR0_ALWAYS_ON \
vm_exit_controls_clearbit(vmx, exit);
}
- int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr)
+ static int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr)
{
unsigned int i;
return -ENOENT;
}
- static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
+ static void vmx_remove_auto_msr(struct vmx_msrs *m, u32 msr,
+ unsigned long vmcs_count_field)
{
int i;
+
+ i = vmx_find_loadstore_msr_slot(m, msr);
+ if (i < 0)
+ return;
+
+ --m->nr;
+ m->val[i] = m->val[m->nr];
+ vmcs_write32(vmcs_count_field, m->nr);
+ }
+
+ static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
+ {
struct msr_autoload *m = &vmx->msr_autoload;
switch (msr) {
}
break;
}
- i = vmx_find_loadstore_msr_slot(&m->guest, msr);
- if (i < 0)
- goto skip_guest;
- --m->guest.nr;
- m->guest.val[i] = m->guest.val[m->guest.nr];
- vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
- skip_guest:
- i = vmx_find_loadstore_msr_slot(&m->host, msr);
- if (i < 0)
- return;
-
- --m->host.nr;
- m->host.val[i] = m->host.val[m->host.nr];
- vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
+ vmx_remove_auto_msr(&m->guest, msr, VM_ENTRY_MSR_LOAD_COUNT);
+ vmx_remove_auto_msr(&m->host, msr, VM_EXIT_MSR_LOAD_COUNT);
}
static __always_inline void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
vm_exit_controls_setbit(vmx, exit);
}
+ static void vmx_add_auto_msr(struct vmx_msrs *m, u32 msr, u64 value,
+ unsigned long vmcs_count_field, struct kvm *kvm)
+ {
+ int i;
+
+ i = vmx_find_loadstore_msr_slot(m, msr);
+ if (i < 0) {
+ if (KVM_BUG_ON(m->nr == MAX_NR_LOADSTORE_MSRS, kvm))
+ return;
+
+ i = m->nr++;
+ m->val[i].index = msr;
+ vmcs_write32(vmcs_count_field, m->nr);
+ }
+ m->val[i].value = value;
+ }
+
static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
- u64 guest_val, u64 host_val, bool entry_only)
+ u64 guest_val, u64 host_val)
{
- int i, j = 0;
struct msr_autoload *m = &vmx->msr_autoload;
+ struct kvm *kvm = vmx->vcpu.kvm;
switch (msr) {
case MSR_EFER:
wrmsrq(MSR_IA32_PEBS_ENABLE, 0);
}
- i = vmx_find_loadstore_msr_slot(&m->guest, msr);
- if (!entry_only)
- j = vmx_find_loadstore_msr_slot(&m->host, msr);
-
- if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) ||
- (j < 0 && m->host.nr == MAX_NR_LOADSTORE_MSRS)) {
- printk_once(KERN_WARNING "Not enough msr switch entries. "
- "Can't add msr %x\n", msr);
- return;
- }
- if (i < 0) {
- i = m->guest.nr++;
- vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
- }
- m->guest.val[i].index = msr;
- m->guest.val[i].value = guest_val;
-
- if (entry_only)
- return;
-
- if (j < 0) {
- j = m->host.nr++;
- vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
- }
- m->host.val[j].index = msr;
- m->host.val[j].value = host_val;
+ vmx_add_auto_msr(&m->guest, msr, guest_val, VM_ENTRY_MSR_LOAD_COUNT, kvm);
+ vmx_add_auto_msr(&m->guest, msr, host_val, VM_EXIT_MSR_LOAD_COUNT, kvm);
}
static bool update_transition_efer(struct vcpu_vmx *vmx)
if (!(guest_efer & EFER_LMA))
guest_efer &= ~EFER_LME;
if (guest_efer != kvm_host.efer)
- add_atomic_switch_msr(vmx, MSR_EFER,
- guest_efer, kvm_host.efer, false);
+ add_atomic_switch_msr(vmx, MSR_EFER, guest_efer, kvm_host.efer);
else
clear_atomic_switch_msr(vmx, MSR_EFER);
return false;
return true;
}
+ static void vmx_add_autostore_msr(struct vcpu_vmx *vmx, u32 msr)
+ {
+ vmx_add_auto_msr(&vmx->msr_autostore, msr, 0, VM_EXIT_MSR_STORE_COUNT,
+ vmx->vcpu.kvm);
+ }
+
+ static void vmx_remove_autostore_msr(struct vcpu_vmx *vmx, u32 msr)
+ {
+ vmx_remove_auto_msr(&vmx->msr_autostore, msr, VM_EXIT_MSR_STORE_COUNT);
+ }
+
#ifdef CONFIG_X86_32
/*
* On 32-bit kernels, VM exits still load the FS and GS bases from the
vmx_prepare_switch_to_host(to_vmx(vcpu));
}
+static void vmx_switch_loaded_vmcs(struct kvm_vcpu *vcpu,
+ struct loaded_vmcs *vmcs)
+{
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+ int cpu;
+
+ cpu = get_cpu();
+ vmx->loaded_vmcs = vmcs;
+ vmx_vcpu_load_vmcs(vcpu, cpu);
+ put_cpu();
+}
+
+static void vmx_load_vmcs01(struct kvm_vcpu *vcpu)
+{
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+
+ if (!is_guest_mode(vcpu)) {
+ WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01);
+ return;
+ }
+
+ WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->nested.vmcs02);
+ vmx_switch_loaded_vmcs(vcpu, &vmx->vmcs01);
+}
+
+static void vmx_put_vmcs01(struct kvm_vcpu *vcpu)
+{
+ if (!is_guest_mode(vcpu))
+ return;
+
+ vmx_switch_loaded_vmcs(vcpu, &to_vmx(vcpu)->nested.vmcs02);
+}
+DEFINE_GUARD(vmx_vmcs01, struct kvm_vcpu *,
+ vmx_load_vmcs01(_T), vmx_put_vmcs01(_T))
+
bool vmx_emulation_required(struct kvm_vcpu *vcpu)
{
return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu);
}
if (nested)
nested_vmx_setup_ctls_msrs(&vmcs_conf, vmx_cap.ept);
+
if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config))) {
- pr_err("Inconsistent VMCS config on CPU %d\n", cpu);
+ u32 *gold = (void *)&vmcs_config;
+ u32 *mine = (void *)&vmcs_conf;
+ int i;
+
+ BUILD_BUG_ON(sizeof(struct vmcs_config) % sizeof(u32));
+
+ pr_err("VMCS config on CPU %d doesn't match reference config:", cpu);
+ for (i = 0; i < sizeof(struct vmcs_config) / sizeof(u32); i++) {
+ if (gold[i] == mine[i])
+ continue;
+
+ pr_cont("\n Offset %u REF = 0x%08x, CPU%u = 0x%08x, mismatch = 0x%08x",
+ i * (int)sizeof(u32), gold[i], cpu, mine[i], gold[i] ^ mine[i]);
+ }
+ pr_cont("\n");
return -EIO;
}
return 0;
}
}
+ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu)
+ {
+ u64 vm_exit_controls_bits = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
+ VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL;
+ bool has_mediated_pmu = kvm_vcpu_has_mediated_pmu(vcpu);
+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+ bool intercept = !has_mediated_pmu;
+ int i;
+
+ if (!enable_mediated_pmu)
+ return;
+
+ if (!cpu_has_save_perf_global_ctrl()) {
+ vm_exit_controls_bits &= ~VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL;
+
+ if (has_mediated_pmu)
+ vmx_add_autostore_msr(vmx, MSR_CORE_PERF_GLOBAL_CTRL);
+ else
+ vmx_remove_autostore_msr(vmx, MSR_CORE_PERF_GLOBAL_CTRL);
+ }
+
+ vm_entry_controls_changebit(vmx, VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
+ has_mediated_pmu);
+
+ vm_exit_controls_changebit(vmx, vm_exit_controls_bits, has_mediated_pmu);
+
+ for (i = 0; i < pmu->nr_arch_gp_counters; i++) {
+ vmx_set_intercept_for_msr(vcpu, MSR_IA32_PERFCTR0 + i,
+ MSR_TYPE_RW, intercept);
+ vmx_set_intercept_for_msr(vcpu, MSR_IA32_PMC0 + i, MSR_TYPE_RW,
+ intercept || !fw_writes_is_enabled(vcpu));
+ }
+ for ( ; i < kvm_pmu_cap.num_counters_gp; i++) {
+ vmx_set_intercept_for_msr(vcpu, MSR_IA32_PERFCTR0 + i,
+ MSR_TYPE_RW, true);
+ vmx_set_intercept_for_msr(vcpu, MSR_IA32_PMC0 + i,
+ MSR_TYPE_RW, true);
+ }
+
+ for (i = 0; i < pmu->nr_arch_fixed_counters; i++)
+ vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_FIXED_CTR0 + i,
+ MSR_TYPE_RW, intercept);
+ for ( ; i < kvm_pmu_cap.num_counters_fixed; i++)
+ vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_FIXED_CTR0 + i,
+ MSR_TYPE_RW, true);
+
+ intercept = kvm_need_perf_global_ctrl_intercept(vcpu);
+ vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_STATUS,
+ MSR_TYPE_RW, intercept);
+ vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL,
+ MSR_TYPE_RW, intercept);
+ vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
+ MSR_TYPE_RW, intercept);
+ }
+
static void vmx_recalc_msr_intercepts(struct kvm_vcpu *vcpu)
{
bool intercept;
vmx_set_intercept_for_msr(vcpu, MSR_IA32_S_CET, MSR_TYPE_RW, intercept);
}
+ vmx_recalc_pmu_msr_intercepts(vcpu);
+
/*
* x2APIC and LBR MSR intercepts are modified on-demand and cannot be
* filtered by userspace.
*/
}
+ static void vmx_recalc_instruction_intercepts(struct kvm_vcpu *vcpu)
+ {
+ exec_controls_changebit(to_vmx(vcpu), CPU_BASED_RDPMC_EXITING,
+ kvm_need_rdpmc_intercept(vcpu));
+ }
+
void vmx_recalc_intercepts(struct kvm_vcpu *vcpu)
{
+ vmx_recalc_instruction_intercepts(vcpu);
vmx_recalc_msr_intercepts(vcpu);
}
vmcs_writel(HOST_SSP, 0);
vmcs_writel(HOST_INTR_SSP_TABLE, 0);
}
+
+ /*
+ * When running a guest with a mediated PMU, guest state is resident in
+ * hardware after VM-Exit. Zero PERF_GLOBAL_CTRL on exit so that host
+ * activity doesn't bleed into the guest counters. When running with
+ * an emulated PMU, PERF_GLOBAL_CTRL is dynamically computed on every
+ * entry/exit to merge guest and host PMU usage.
+ */
+ if (enable_mediated_pmu)
+ vmcs_write64(HOST_IA32_PERF_GLOBAL_CTRL, 0);
}
void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
VM_EXIT_CLEAR_IA32_RTIT_CTL);
/* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */
return vmexit_ctrl &
- ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER);
+ ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER |
+ VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL);
}
void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
- if (is_guest_mode(vcpu)) {
- vmx->nested.update_vmcs01_apicv_status = true;
- return;
- }
+ guard(vmx_vmcs01)(vcpu);
pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
vmcs_write64(VM_FUNCTION_CONTROL, 0);
vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
+ vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.val));
vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
!kvm_is_cr0_bit_set(vcpu, X86_CR0_TS);
}
+static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
+{
+ unsigned long cr2 = vmx_get_exit_qual(vcpu);
+
+ if (vcpu->arch.apf.host_apf_flags)
+ goto handle_pf;
+
+ /* When using EPT, KVM intercepts #PF only to detect illegal GPAs. */
+ WARN_ON_ONCE(enable_ept && !allow_smaller_maxphyaddr);
+
+ /*
+ * On SGX2 hardware, EPCM violations are delivered as #PF with the SGX
+ * flag set in the error code (SGX1 hardware generates #GP(0)). EPCM
+ * violations have nothing to do with shadow paging and can never be
+ * resolved by KVM; always reflect them into the guest.
+ */
+ if (error_code & PFERR_SGX_MASK) {
+ WARN_ON_ONCE(!IS_ENABLED(CONFIG_X86_SGX_KVM) ||
+ !cpu_feature_enabled(X86_FEATURE_SGX2));
+
+ if (guest_cpu_cap_has(vcpu, X86_FEATURE_SGX2))
+ kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code);
+ else
+ kvm_inject_gp(vcpu, 0);
+ return 1;
+ }
+
+ /*
+ * If EPT is enabled, fixup and inject the #PF. KVM intercepts #PFs
+ * only to set PFERR_RSVD as appropriate (hardware won't set RSVD due
+ * to the GPA being legal with respect to host.MAXPHYADDR).
+ */
+ if (enable_ept) {
+ kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code);
+ return 1;
+ }
+
+handle_pf:
+ return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
+}
+
static int handle_exception_nmi(struct kvm_vcpu *vcpu)
{
struct vcpu_vmx *vmx = to_vmx(vcpu);
struct kvm_run *kvm_run = vcpu->run;
u32 intr_info, ex_no, error_code;
- unsigned long cr2, dr6;
+ unsigned long dr6;
u32 vect_info;
vect_info = vmx->idt_vectoring_info;
return 0;
}
- if (is_page_fault(intr_info)) {
- cr2 = vmx_get_exit_qual(vcpu);
- if (enable_ept && !vcpu->arch.apf.host_apf_flags) {
- /*
- * EPT will cause page fault only if we need to
- * detect illegal GPAs.
- */
- WARN_ON_ONCE(!allow_smaller_maxphyaddr);
- kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code);
- return 1;
- } else
- return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
- }
+ if (is_page_fault(intr_info))
+ return vmx_handle_page_fault(vcpu, error_code);
ex_no = intr_info & INTR_INFO_VECTOR_MASK;
vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX);
}
+static void nested_vmx_mark_all_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
+{
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+
+ kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.apic_access_page_map);
+ kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.virtual_apic_map);
+ kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.pi_desc_map);
+}
+
static void vmx_dump_sel(char *name, uint32_t sel)
{
pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
if (vmcs_read32(VM_ENTRY_MSR_LOAD_COUNT) > 0)
vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest);
if (vmcs_read32(VM_EXIT_MSR_STORE_COUNT) > 0)
- vmx_dump_msrs("guest autostore", &vmx->msr_autostore.guest);
+ vmx_dump_msrs("autostore", &vmx->msr_autostore);
if (vmentry_ctl & VM_ENTRY_LOAD_CET_STATE)
pr_err("S_CET = 0x%016lx, SSP = 0x%016lx, SSP TABLE = 0x%016lx\n",
* Mark them dirty on every exit from L2 to prevent them from
* getting out of sync with dirty tracking.
*/
- nested_mark_vmcs12_pages_dirty(vcpu);
+ nested_vmx_mark_all_vmcs12_pages_dirty(vcpu);
/*
* Synthesize a triple fault if L2 state is invalid. In normal
nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
return;
+ guard(vmx_vmcs01)(vcpu);
+
tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr;
- if (is_guest_mode(vcpu))
- to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold;
- else
- vmcs_write32(TPR_THRESHOLD, tpr_threshold);
+ vmcs_write32(TPR_THRESHOLD, tpr_threshold);
}
void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
!cpu_has_vmx_virtualize_x2apic_mode())
return;
- /* Postpone execution until vmcs01 is the current VMCS. */
- if (is_guest_mode(vcpu)) {
- vmx->nested.change_vmcs01_virtual_apic_mode = true;
- return;
- }
+ guard(vmx_vmcs01)(vcpu);
sec_exec_control = secondary_exec_controls_get(vmx);
sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
* only do so if its physical address has changed, but
* the guest may have inserted a non-APIC mapping into
* the TLB while the APIC access page was disabled.
+ *
+ * If L2 is active, immediately flush L1's TLB instead
+ * of requesting a flush of the current TLB, because
+ * the current TLB context is L2's.
*/
- kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
+ if (!is_guest_mode(vcpu))
+ kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
+ else if (!enable_ept)
+ vpid_sync_context(vmx->vpid);
+ else if (VALID_PAGE(vcpu->arch.root_mmu.root.hpa))
+ vmx_flush_tlb_ept_root(vcpu->arch.root_mmu.root.hpa);
}
break;
case LAPIC_MODE_X2APIC:
kvm_pfn_t pfn;
bool writable;
- /* Defer reload until vmcs01 is the current VMCS. */
- if (is_guest_mode(vcpu)) {
- to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true;
- return;
- }
+ /* Note, the VIRTUALIZE_APIC_ACCESSES check needs to query vmcs01. */
+ guard(vmx_vmcs01)(vcpu);
if (!(secondary_exec_controls_get(to_vmx(vcpu)) &
SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
u16 status;
u8 old;
- /*
- * If L2 is active, defer the SVI update until vmcs01 is loaded, as SVI
- * is only relevant for if and only if Virtual Interrupt Delivery is
- * enabled in vmcs12, and if VID is enabled then L2 EOIs affect L2's
- * vAPIC, not L1's vAPIC. KVM must update vmcs01 on the next nested
- * VM-Exit, otherwise L1 with run with a stale SVI.
- */
- if (is_guest_mode(vcpu)) {
- /*
- * KVM is supposed to forward intercepted L2 EOIs to L1 if VID
- * is enabled in vmcs12; as above, the EOIs affect L2's vAPIC.
- * Note, userspace can stuff state while L2 is active; assert
- * that VID is disabled if and only if the vCPU is in KVM_RUN
- * to avoid false positives if userspace is setting APIC state.
- */
- WARN_ON_ONCE(vcpu->wants_to_run &&
- nested_cpu_has_vid(get_vmcs12(vcpu)));
- to_vmx(vcpu)->nested.update_vmcs01_hwapic_isr = true;
- return;
- }
-
if (max_isr == -1)
max_isr = 0;
+ /*
+ * Always update SVI in vmcs01, as SVI is only relevant for L2 if and
+ * only if Virtual Interrupt Delivery is enabled in vmcs12, and if VID
+ * is enabled then L2 EOIs affect L2's vAPIC, not L1's vAPIC.
+ */
+ guard(vmx_vmcs01)(vcpu);
+
status = vmcs_read16(GUEST_INTR_STATUS);
old = status >> 8;
if (max_isr != old) {
struct perf_guest_switch_msr *msrs;
struct kvm_pmu *pmu = vcpu_to_pmu(&vmx->vcpu);
+ if (kvm_vcpu_has_mediated_pmu(&vmx->vcpu))
+ return;
+
pmu->host_cross_mapped_mask = 0;
if (pmu->pebs_enable & pmu->global_ctrl)
intel_pmu_cross_mapped_check(pmu);
clear_atomic_switch_msr(vmx, msrs[i].msr);
else
add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
- msrs[i].host, false);
+ msrs[i].host);
+ }
+
+ static void vmx_refresh_guest_perf_global_control(struct kvm_vcpu *vcpu)
+ {
+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
+ struct vcpu_vmx *vmx = to_vmx(vcpu);
+
+ if (msr_write_intercepted(vmx, MSR_CORE_PERF_GLOBAL_CTRL))
+ return;
+
+ if (!cpu_has_save_perf_global_ctrl()) {
+ int slot = vmx_find_loadstore_msr_slot(&vmx->msr_autostore,
+ MSR_CORE_PERF_GLOBAL_CTRL);
+
+ if (WARN_ON_ONCE(slot < 0))
+ return;
+
+ pmu->global_ctrl = vmx->msr_autostore.val[slot].value;
+ vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, pmu->global_ctrl);
+ return;
+ }
+
+ pmu->global_ctrl = vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL);
}
static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit)
vmx->loaded_vmcs->launched = 1;
+ vmx_refresh_guest_perf_global_control(vcpu);
+
vmx_recover_nmi_blocking(vmx);
vmx_complete_interrupts(vmx);
if (boot_cpu_has(X86_FEATURE_PDCM))
rdmsrq(MSR_IA32_PERF_CAPABILITIES, host_perf_cap);
- if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) {
+ if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR) &&
+ !enable_mediated_pmu) {
x86_perf_get_lbr(&vmx_lbr_caps);
/*
static __init void vmx_set_cpu_caps(void)
{
- kvm_set_cpu_caps();
+ kvm_initialize_cpu_caps();
/* CPUID 0x1 */
if (nested)
kvm_cpu_cap_clear(X86_FEATURE_SHSTK);
kvm_cpu_cap_clear(X86_FEATURE_IBT);
}
+
+ kvm_setup_xss_caps();
+ kvm_finalize_cpu_caps();
}
static bool vmx_is_io_intercepted(struct kvm_vcpu *vcpu,
if (WARN_ON_ONCE(!enable_pml))
return;
- if (is_guest_mode(vcpu)) {
- vmx->nested.update_vmcs01_cpu_dirty_logging = true;
- return;
- }
+ guard(vmx_vmcs01)(vcpu);
/*
* Note, nr_memslots_dirty_logging can be changed concurrent with this
* can hide/show features based on kvm_cpu_cap_has().
*/
if (nested) {
- nested_vmx_setup_ctls_msrs(&vmcs_config, vmx_capability.ept);
-
r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers);
if (r)
return r;
}
r = alloc_kvm_area();
- if (r && nested)
- nested_vmx_hardware_unsetup();
+ if (r)
+ goto err_kvm_area;
kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);
kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
+ return 0;
+
+err_kvm_area:
+ if (nested)
+ nested_vmx_hardware_unsetup();
return r;
}
*/
bool vmcs02_initialized;
- bool change_vmcs01_virtual_apic_mode;
- bool reload_vmcs01_apic_access_page;
- bool update_vmcs01_cpu_dirty_logging;
- bool update_vmcs01_apicv_status;
- bool update_vmcs01_hwapic_isr;
-
/*
* Enlightened VMCS has been enabled. It does not mean that L1 has to
* use it. However, VMX features available to L1 will be limited based
u64 pre_vmenter_ssp;
u64 pre_vmenter_ssp_tbl;
- /* to migrate it to L1 if L2 writes to L1's CR8 directly */
- int l1_tpr_threshold;
-
u16 vpid02;
u16 last_vpid;
+ int tsc_autostore_slot;
struct nested_vmx_msrs msrs;
/* SMM related state */
struct vmx_msrs host;
} msr_autoload;
- struct msr_autostore {
- struct vmx_msrs guest;
- } msr_autostore;
+ struct vmx_msrs msr_autostore;
struct {
int vm86_active;
unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx);
bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs,
unsigned int flags);
- int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr);
void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);
void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool set);
VM_EXIT_CLEAR_BNDCFGS | \
VM_EXIT_PT_CONCEAL_PIP | \
VM_EXIT_CLEAR_IA32_RTIT_CTL | \
- VM_EXIT_LOAD_CET_STATE)
+ VM_EXIT_LOAD_CET_STATE | \
+ VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL)
#define KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL \
(PIN_BASED_EXT_INTR_MASK | \
#define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE
-#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
- KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
+#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
+ KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK | \
+ KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \
+ KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
static void process_nmi(struct kvm_vcpu *vcpu);
EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_pmu);
module_param(enable_pmu, bool, 0444);
+ /* Enable/disabled mediated PMU virtualization. */
+ bool __read_mostly enable_mediated_pmu;
+ EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_mediated_pmu);
+
bool __read_mostly eager_page_split = true;
module_param(eager_page_split, bool, 0644);
fastpath_t handle_fastpath_invd(struct kvm_vcpu *vcpu)
{
+ if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
+ return EXIT_FASTPATH_NONE;
+
if (!kvm_emulate_invd(vcpu))
return EXIT_FASTPATH_EXIT_USERSPACE;
static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
+ if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
+ return EXIT_FASTPATH_NONE;
+
switch (msr) {
case APIC_BASE_MSR + (APIC_ICR >> 4):
if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic) ||
u64 val;
/*
- * Disallow writes to immutable feature MSRs after KVM_RUN. KVM does
- * not support modifying the guest vCPU model on the fly, e.g. changing
- * the nVMX capabilities while L2 is running is nonsensical. Allow
- * writes of the same value, e.g. to allow userspace to blindly stuff
- * all MSRs when emulating RESET.
+ * Reject writes to immutable feature MSRs if the vCPU model is frozen,
+ * as KVM doesn't support modifying the guest vCPU model on the fly,
+ * e.g. changing the VMX capabilities MSRs while L2 is active is
+ * nonsensical. Allow writes of the same value, e.g. so that userspace
+ * can blindly stuff all MSRs when emulating RESET.
*/
- if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index) &&
+ if (!kvm_can_set_cpuid_and_feature_msrs(vcpu) &&
+ kvm_is_immutable_feature_msr(index) &&
(do_get_msr(vcpu, index, &val) || *data != val))
return -EINVAL;
vcpu->arch.perf_capabilities = data;
kvm_pmu_refresh(vcpu);
+ kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu);
break;
case MSR_IA32_PRED_CMD: {
u64 reserved_bits = ~(PRED_CMD_IBPB | PRED_CMD_SBPB);
break;
case MSR_KVM_WALL_CLOCK_NEW:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
vcpu->kvm->arch.wall_clock = data;
kvm_write_wall_clock(vcpu->kvm, data, 0);
break;
case MSR_KVM_WALL_CLOCK:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
vcpu->kvm->arch.wall_clock = data;
kvm_write_wall_clock(vcpu->kvm, data, 0);
break;
case MSR_KVM_SYSTEM_TIME_NEW:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
break;
case MSR_KVM_SYSTEM_TIME:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
kvm_write_system_time(vcpu, data, true, msr_info->host_initiated);
break;
case MSR_KVM_ASYNC_PF_EN:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (kvm_pv_enable_async_pf(vcpu, data))
return 1;
break;
case MSR_KVM_ASYNC_PF_INT:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (kvm_pv_enable_async_pf_int(vcpu, data))
return 1;
break;
case MSR_KVM_ASYNC_PF_ACK:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (data & 0x1) {
/*
* Pairs with the smp_mb__after_atomic() in
break;
case MSR_KVM_STEAL_TIME:
if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (unlikely(!sched_info_on()))
return 1;
break;
case MSR_KVM_PV_EOI_EN:
if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
return 1;
case MSR_KVM_POLL_CONTROL:
if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
/* only enable bit supported */
if (data & (-1ULL << 1))
break;
case MSR_KVM_WALL_CLOCK:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->kvm->arch.wall_clock;
break;
case MSR_KVM_WALL_CLOCK_NEW:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->kvm->arch.wall_clock;
break;
case MSR_KVM_SYSTEM_TIME:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.time;
break;
case MSR_KVM_SYSTEM_TIME_NEW:
if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.time;
break;
case MSR_KVM_ASYNC_PF_EN:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.apf.msr_en_val;
break;
case MSR_KVM_ASYNC_PF_INT:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.apf.msr_int_val;
break;
case MSR_KVM_ASYNC_PF_ACK:
if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = 0;
break;
case MSR_KVM_STEAL_TIME:
if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.st.msr_val;
break;
case MSR_KVM_PV_EOI_EN:
if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.pv_eoi.msr_val;
break;
case MSR_KVM_POLL_CONTROL:
if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
- return 1;
+ return KVM_MSR_RET_UNSUPPORTED;
msr_info->data = vcpu->arch.msr_kvm_poll_control;
break;
break;
case KVM_CAP_X2APIC_API:
r = KVM_X2APIC_API_VALID_FLAGS;
+ if (kvm && !irqchip_split(kvm))
+ r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST;
break;
case KVM_CAP_NESTED_STATE:
r = kvm_x86_ops.nested_ops->get_state ?
static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
struct kvm_xsave *guest_xsave)
{
+ union fpregs_state *xstate = (union fpregs_state *)guest_xsave->region;
+
if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
+ /*
+ * For backwards compatibility, do not expect disabled features to be in
+ * their initial state. XSTATE_BV[i] must still be cleared whenever
+ * XFD[i]=1, or XRSTOR would cause a #NM.
+ */
+ xstate->xsave.header.xfeatures &= ~vcpu->arch.guest_fpu.fpstate->xfd;
+
return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
guest_xsave->region,
kvm_caps.supported_xcr0,
case KVM_CAP_SPLIT_IRQCHIP: {
mutex_lock(&kvm->lock);
r = -EINVAL;
- if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
+ if (cap->args[0] > KVM_MAX_IRQ_ROUTES)
goto split_irqchip_unlock;
r = -EEXIST;
if (irqchip_in_kernel(kvm))
if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
break;
+ if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) &&
+ (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST))
+ break;
+
+ if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) &&
+ !irqchip_split(kvm))
+ break;
+
if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
kvm->arch.x2apic_format = true;
if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
kvm->arch.x2apic_broadcast_quirk_disabled = true;
+ if (cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST)
+ kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_ENABLED;
+ if (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)
+ kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_DISABLED;
+
r = 0;
break;
case KVM_CAP_X86_DISABLE_EXITS:
break;
mutex_lock(&kvm->lock);
- if (!kvm->created_vcpus) {
+ if (!kvm->created_vcpus && !kvm->arch.created_mediated_pmu) {
kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
r = 0;
}
};
#endif
+void kvm_setup_xss_caps(void)
+{
+ if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
+ kvm_caps.supported_xss = 0;
+
+ if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) &&
+ !kvm_cpu_cap_has(X86_FEATURE_IBT))
+ kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL;
+
+ if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) {
+ kvm_cpu_cap_clear(X86_FEATURE_SHSTK);
+ kvm_cpu_cap_clear(X86_FEATURE_IBT);
+ kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL;
+ }
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps);
+
static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
{
memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
#endif
- kvm_register_perf_callbacks(ops->handle_intel_pt_intr);
+ __kvm_register_perf_callbacks(ops->handle_intel_pt_intr,
+ enable_mediated_pmu ? kvm_handle_guest_mediated_pmi : NULL);
if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && tdp_mmu_enabled)
kvm_caps.supported_vm_types |= BIT(KVM_X86_SW_PROTECTED_VM);
if (!tdp_enabled)
kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
- if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
- kvm_caps.supported_xss = 0;
-
- if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) &&
- !kvm_cpu_cap_has(X86_FEATURE_IBT))
- kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL;
-
- if ((kvm_caps.supported_xss & XFEATURE_MASK_CET_ALL) != XFEATURE_MASK_CET_ALL) {
- kvm_cpu_cap_clear(X86_FEATURE_SHSTK);
- kvm_cpu_cap_clear(X86_FEATURE_IBT);
- kvm_caps.supported_xss &= ~XFEATURE_MASK_CET_ALL;
- }
-
if (kvm_caps.has_tsc_control) {
/*
* Make sure the user can only configure tsc_khz values that
.dest_id = apicid,
};
- kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
+ kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq);
}
bool kvm_apicv_activated(struct kvm *kvm)
run_flags |= KVM_RUN_LOAD_DEBUGCTL;
vcpu->arch.host_debugctl = debug_ctl;
+ kvm_mediated_pmu_load(vcpu);
+
guest_timing_enter_irqoff();
/*
kvm_load_host_pkru(vcpu);
+ kvm_mediated_pmu_put(vcpu);
+
/*
* Do this here before restoring debug registers on the host. And
* since we do this before handling the vmexit, a DR access vmexit
if (is_guest_mode(vcpu)) {
int r = kvm_check_nested_events(vcpu);
- WARN_ON_ONCE(r == -EBUSY);
- if (r < 0)
+ if (r < 0 && r != -EBUSY)
return 0;
}
fastpath_t handle_fastpath_hlt(struct kvm_vcpu *vcpu)
{
+ if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
+ return EXIT_FASTPATH_NONE;
+
if (!kvm_emulate_halt(vcpu))
return EXIT_FASTPATH_EXIT_USERSPACE;
return;
if (is_pae_paging(vcpu)) {
+ kvm_vcpu_srcu_read_lock(vcpu);
for (i = 0 ; i < 4 ; i++)
sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
+ kvm_vcpu_srcu_read_unlock(vcpu);
}
}
return 0;
}
+ #define PERF_MEDIATED_PMU_MSG \
+ "Failed to enable mediated vPMU, try disabling system wide perf events and nmi_watchdog.\n"
+
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
{
+ int r;
+
if (kvm_check_tsc_unstable() && kvm->created_vcpus)
pr_warn_once("SMP vm created on host with unstable TSC; "
"guest TSC will not be reliable\n");
if (id >= kvm->arch.max_vcpu_ids)
return -EINVAL;
- return kvm_x86_call(vcpu_precreate)(kvm);
+ /*
+ * Note, any actions done by .vcpu_create() must be idempotent with
+ * respect to creating multiple vCPUs, and therefore are not undone if
+ * creating a vCPU fails (including failure during pre-create).
+ */
+ r = kvm_x86_call(vcpu_precreate)(kvm);
+ if (r)
+ return r;
+
+ if (enable_mediated_pmu && kvm->arch.enable_pmu &&
+ !kvm->arch.created_mediated_pmu) {
+ if (irqchip_in_kernel(kvm)) {
+ r = perf_create_mediated_pmu();
+ if (r) {
+ pr_warn_ratelimited(PERF_MEDIATED_PMU_MSG);
+ return r;
+ }
+ kvm->arch.created_mediated_pmu = true;
+ } else {
+ kvm->arch.enable_pmu = false;
+ }
+ }
+ return 0;
}
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
#endif
kvm_mmu_pre_destroy_vm(kvm);
- static_call_cond(kvm_x86_vm_pre_destroy)(kvm);
+ kvm_x86_call(vm_pre_destroy)(kvm);
}
void kvm_arch_destroy_vm(struct kvm *kvm)
__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
mutex_unlock(&kvm->slots_lock);
}
+ if (kvm->arch.created_mediated_pmu)
+ perf_release_mediated_pmu();
kvm_destroy_vcpus(kvm);
kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
#ifdef CONFIG_KVM_IOAPIC
return 1;
}
+ /*
+ * When ERAPS is supported, invalidating a specific PCID clears
+ * the RAP (Return Address Predicator).
+ */
+ if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
+ kvm_register_is_dirty(vcpu, VCPU_EXREG_ERAPS);
+
kvm_invalidate_pcid(vcpu, operand.pcid);
return kvm_skip_emulated_instruction(vcpu);
fallthrough;
case INVPCID_TYPE_ALL_INCL_GLOBAL:
+ /*
+ * Don't bother marking VCPU_EXREG_ERAPS dirty, SVM will take
+ * care of doing so when emulating the full guest TLB flush
+ * (the RAP is cleared on all implicit TLB flushes).
+ */
kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
return kvm_skip_emulated_instruction(vcpu);
indirect_branch_prediction_barrier();
}
-static inline bool kvm_vcpu_has_run(struct kvm_vcpu *vcpu)
+/*
+ * Disallow modifying CPUID and feature MSRs, which affect the core virtual CPU
+ * model exposed to the guest and virtualized by KVM, if the vCPU has already
+ * run or is in guest mode (L2). In both cases, KVM has already consumed the
+ * current virtual CPU model, and doesn't support "unwinding" to react to the
+ * new model.
+ *
+ * Note, the only way is_guest_mode() can be true with 'last_vmentry_cpu == -1'
+ * is if userspace sets CPUID and feature MSRs (to enable VMX/SVM), then sets
+ * nested state, and then attempts to set CPUID and/or feature MSRs *again*.
+ */
+static inline bool kvm_can_set_cpuid_and_feature_msrs(struct kvm_vcpu *vcpu)
{
- return vcpu->arch.last_vmentry_cpu != -1;
+ return vcpu->arch.last_vmentry_cpu == -1 && !is_guest_mode(vcpu);
}
static inline void kvm_set_mp_state(struct kvm_vcpu *vcpu, int mp_state)
extern struct kvm_host_values kvm_host;
extern bool enable_pmu;
+ extern bool enable_mediated_pmu;
+void kvm_setup_xss_caps(void);
+
/*
* Get a filtered version of KVM's supported XCR0 that strips out dynamic
* features for which the current process doesn't (yet) have permission to use.
unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn);
void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn);
void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
+void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn);
int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map,
bool writable);
return __kvm_vcpu_map(vcpu, gpa, map, false);
}
+static inline void kvm_vcpu_map_mark_dirty(struct kvm_vcpu *vcpu,
+ struct kvm_host_map *map)
+{
+ if (kvm_vcpu_mapped(map))
+ kvm_vcpu_mark_page_dirty(vcpu, map->gfn);
+}
+
unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn);
unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable);
int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset,
int offset, int len);
int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
unsigned long len);
-void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn);
/**
* kvm_gpc_init - initialize gfn_to_pfn_cache.
#ifdef CONFIG_GUEST_PERF_EVENTS
unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu);
- void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void));
+ void __kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void),
+ void (*mediated_pmi_handler)(void));
+
+ static inline void kvm_register_perf_callbacks(void)
+ {
+ __kvm_register_perf_callbacks(NULL, NULL);
+ }
+
void kvm_unregister_perf_callbacks(void);
#else
- static inline void kvm_register_perf_callbacks(void *ign) {}
+ static inline void kvm_register_perf_callbacks(void) {}
static inline void kvm_unregister_perf_callbacks(void) {}
#endif /* CONFIG_GUEST_PERF_EVENTS */
* @gfn: starting GFN to be populated
* @src: userspace-provided buffer containing data to copy into GFN range
* (passed to @post_populate, and incremented on each iteration
- * if not NULL)
+ * if not NULL). Must be page-aligned.
* @npages: number of pages to copy from userspace-buffer
* @post_populate: callback to issue for each gmem page that backs the GPA
* range
* Returns the number of pages that were populated.
*/
typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
- void __user *src, int order, void *opaque);
+ struct page *page, void *opaque);
long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages,
kvm_gmem_populate_cb post_populate, void *opaque);
/*
* Performance events:
*
- * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
+ * Copyright (C) 2008-2009, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
* Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
* Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
*
#define PERF_PMU_CAP_EXTENDED_HW_TYPE 0x0100
#define PERF_PMU_CAP_AUX_PAUSE 0x0200
#define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400
+ #define PERF_PMU_CAP_MEDIATED_VPMU 0x0800
/**
* pmu::scope
u64 index;
};
+ struct perf_time_ctx {
+ u64 time;
+ u64 stamp;
+ u64 offset;
+ };
/**
* struct perf_event_context - event context structure
/*
* Context clock, runs when context enabled.
*/
- u64 time;
- u64 timestamp;
- u64 timeoffset;
+ struct perf_time_ctx time;
+
+ /*
+ * Context clock, runs when in the guest mode.
+ */
+ struct perf_time_ctx timeguest;
/*
* These fields let us detect when two contexts have both
* This is a per-cpu dynamically allocated data structure.
*/
struct perf_cgroup_info {
- u64 time;
- u64 timestamp;
- u64 timeoffset;
+ struct perf_time_ctx time;
+ struct perf_time_ctx timeguest;
int active;
};
unsigned int (*state)(void);
unsigned long (*get_ip)(void);
unsigned int (*handle_intel_pt_intr)(void);
+
+ void (*handle_mediated_pmi)(void);
};
#ifdef CONFIG_GUEST_PERF_EVENTS
DECLARE_STATIC_CALL(__perf_guest_state, *perf_guest_cbs->state);
DECLARE_STATIC_CALL(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
DECLARE_STATIC_CALL(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
+ DECLARE_STATIC_CALL(__perf_guest_handle_mediated_pmi, *perf_guest_cbs->handle_mediated_pmi);
static inline unsigned int perf_guest_state(void)
{
return static_call(__perf_guest_handle_intel_pt_intr)();
}
+ static inline void perf_guest_handle_mediated_pmi(void)
+ {
+ static_call(__perf_guest_handle_mediated_pmi)();
+ }
+
extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);
extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs);
extern int perf_event_period(struct perf_event *event, u64 value);
extern u64 perf_event_pause(struct perf_event *event, bool reset);
+ #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
+ int perf_create_mediated_pmu(void);
+ void perf_release_mediated_pmu(void);
+ void perf_load_guest_context(void);
+ void perf_put_guest_context(void);
+ #endif
+
#else /* !CONFIG_PERF_EVENTS: */
static inline void *
/*
* Performance events core code:
*
- * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
+ * Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
* Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
* Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
* Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
#include <linux/task_work.h>
#include <linux/percpu-rwsem.h>
#include <linux/unwind_deferred.h>
+ #include <linux/kvm_types.h>
#include "internal.h"
EVENT_CPU = 0x10,
EVENT_CGROUP = 0x20,
+ /*
+ * EVENT_GUEST is set when scheduling in/out events between the host
+ * and a guest with a mediated vPMU. Among other things, EVENT_GUEST
+ * is used:
+ *
+ * - In for_each_epc() to skip PMUs that don't support events in a
+ * MEDIATED_VPMU guest, i.e. don't need to be context switched.
+ * - To indicate the start/end point of the events in a guest. Guest
+ * running time is deducted for host-only (exclude_guest) events.
+ */
+ EVENT_GUEST = 0x40,
+ EVENT_FLAGS = EVENT_CGROUP | EVENT_GUEST,
/* compound helpers */
EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
EVENT_TIME_FROZEN = EVENT_TIME | EVENT_FROZEN,
static cpumask_var_t perf_online_sys_mask;
static struct kmem_cache *perf_event_cache;
+ #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
+ static DEFINE_PER_CPU(bool, guest_ctx_loaded);
+
+ static __always_inline bool is_guest_mediated_pmu_loaded(void)
+ {
+ return __this_cpu_read(guest_ctx_loaded);
+ }
+ #else
+ static __always_inline bool is_guest_mediated_pmu_loaded(void)
+ {
+ return false;
+ }
+ #endif
+
/*
* perf event paranoia level:
* -1 - not paranoid at all
___p; \
})
- #define for_each_epc(_epc, _ctx, _pmu, _cgroup) \
+ static bool perf_skip_pmu_ctx(struct perf_event_pmu_context *pmu_ctx,
+ enum event_type_t event_type)
+ {
+ if ((event_type & EVENT_CGROUP) && !pmu_ctx->nr_cgroups)
+ return true;
+ if ((event_type & EVENT_GUEST) &&
+ !(pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU))
+ return true;
+ return false;
+ }
+
+ #define for_each_epc(_epc, _ctx, _pmu, _event_type) \
list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \
- if (_cgroup && !_epc->nr_cgroups) \
+ if (perf_skip_pmu_ctx(_epc, _event_type)) \
continue; \
else if (_pmu && _epc->pmu != _pmu) \
continue; \
else
- static void perf_ctx_disable(struct perf_event_context *ctx, bool cgroup)
+ static void perf_ctx_disable(struct perf_event_context *ctx,
+ enum event_type_t event_type)
{
struct perf_event_pmu_context *pmu_ctx;
- for_each_epc(pmu_ctx, ctx, NULL, cgroup)
+ for_each_epc(pmu_ctx, ctx, NULL, event_type)
perf_pmu_disable(pmu_ctx->pmu);
}
- static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup)
+ static void perf_ctx_enable(struct perf_event_context *ctx,
+ enum event_type_t event_type)
{
struct perf_event_pmu_context *pmu_ctx;
- for_each_epc(pmu_ctx, ctx, NULL, cgroup)
+ for_each_epc(pmu_ctx, ctx, NULL, event_type)
perf_pmu_enable(pmu_ctx->pmu);
}
static void ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type);
static void ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type);
+ static inline void update_perf_time_ctx(struct perf_time_ctx *time, u64 now, bool adv)
+ {
+ if (adv)
+ time->time += now - time->stamp;
+ time->stamp = now;
+
+ /*
+ * The above: time' = time + (now - timestamp), can be re-arranged
+ * into: time` = now + (time - timestamp), which gives a single value
+ * offset to compute future time without locks on.
+ *
+ * See perf_event_time_now(), which can be used from NMI context where
+ * it's (obviously) not possible to acquire ctx->lock in order to read
+ * both the above values in a consistent manner.
+ */
+ WRITE_ONCE(time->offset, time->time - time->stamp);
+ }
+
+ static_assert(offsetof(struct perf_event_context, timeguest) -
+ offsetof(struct perf_event_context, time) ==
+ sizeof(struct perf_time_ctx));
+
+ #define T_TOTAL 0
+ #define T_GUEST 1
+
+ static inline u64 __perf_event_time_ctx(struct perf_event *event,
+ struct perf_time_ctx *times)
+ {
+ u64 time = times[T_TOTAL].time;
+
+ if (event->attr.exclude_guest)
+ time -= times[T_GUEST].time;
+
+ return time;
+ }
+
+ static inline u64 __perf_event_time_ctx_now(struct perf_event *event,
+ struct perf_time_ctx *times,
+ u64 now)
+ {
+ if (is_guest_mediated_pmu_loaded() && event->attr.exclude_guest) {
+ /*
+ * (now + times[total].offset) - (now + times[guest].offset) :=
+ * times[total].offset - times[guest].offset
+ */
+ return READ_ONCE(times[T_TOTAL].offset) - READ_ONCE(times[T_GUEST].offset);
+ }
+
+ return now + READ_ONCE(times[T_TOTAL].offset);
+ }
+
#ifdef CONFIG_CGROUP_PERF
static inline bool
return event->cgrp != NULL;
}
+ static_assert(offsetof(struct perf_cgroup_info, timeguest) -
+ offsetof(struct perf_cgroup_info, time) ==
+ sizeof(struct perf_time_ctx));
+
static inline u64 perf_cgroup_event_time(struct perf_event *event)
{
struct perf_cgroup_info *t;
t = per_cpu_ptr(event->cgrp->info, event->cpu);
- return t->time;
+ return __perf_event_time_ctx(event, &t->time);
}
static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
t = per_cpu_ptr(event->cgrp->info, event->cpu);
if (!__load_acquire(&t->active))
- return t->time;
- now += READ_ONCE(t->timeoffset);
- return now;
+ return __perf_event_time_ctx(event, &t->time);
+
+ return __perf_event_time_ctx_now(event, &t->time, now);
}
- static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv)
+ static inline void __update_cgrp_guest_time(struct perf_cgroup_info *info, u64 now, bool adv)
{
- if (adv)
- info->time += now - info->timestamp;
- info->timestamp = now;
- /*
- * see update_context_time()
- */
- WRITE_ONCE(info->timeoffset, info->time - info->timestamp);
+ update_perf_time_ctx(&info->timeguest, now, adv);
+ }
+
+ static inline void update_cgrp_time(struct perf_cgroup_info *info, u64 now)
+ {
+ update_perf_time_ctx(&info->time, now, true);
+ if (is_guest_mediated_pmu_loaded())
+ __update_cgrp_guest_time(info, now, true);
}
static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
cgrp = container_of(css, struct perf_cgroup, css);
info = this_cpu_ptr(cgrp->info);
- __update_cgrp_time(info, now, true);
+ update_cgrp_time(info, now);
if (final)
__store_release(&info->active, 0);
}
* Do not update time when cgroup is not active
*/
if (info->active)
- __update_cgrp_time(info, perf_clock(), true);
+ update_cgrp_time(info, perf_clock());
}
static inline void
- perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
+ perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest)
{
struct perf_event_context *ctx = &cpuctx->ctx;
struct perf_cgroup *cgrp = cpuctx->cgrp;
for (css = &cgrp->css; css; css = css->parent) {
cgrp = container_of(css, struct perf_cgroup, css);
info = this_cpu_ptr(cgrp->info);
- __update_cgrp_time(info, ctx->timestamp, false);
- __store_release(&info->active, 1);
+ if (guest) {
+ __update_cgrp_guest_time(info, ctx->time.stamp, false);
+ } else {
+ update_perf_time_ctx(&info->time, ctx->time.stamp, false);
+ __store_release(&info->active, 1);
+ }
}
}
return;
WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
-
- perf_ctx_disable(&cpuctx->ctx, true);
+ perf_ctx_disable(&cpuctx->ctx, EVENT_CGROUP);
ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);
/*
*/
ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);
- perf_ctx_enable(&cpuctx->ctx, true);
+ perf_ctx_enable(&cpuctx->ctx, EVENT_CGROUP);
}
static int perf_cgroup_ensure_storage(struct perf_event *event,
}
static inline void
- perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
+ perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest)
{
}
*/
static void __update_context_time(struct perf_event_context *ctx, bool adv)
{
- u64 now = perf_clock();
-
lockdep_assert_held(&ctx->lock);
- if (adv)
- ctx->time += now - ctx->timestamp;
- ctx->timestamp = now;
+ update_perf_time_ctx(&ctx->time, perf_clock(), adv);
+ }
- /*
- * The above: time' = time + (now - timestamp), can be re-arranged
- * into: time` = now + (time - timestamp), which gives a single value
- * offset to compute future time without locks on.
- *
- * See perf_event_time_now(), which can be used from NMI context where
- * it's (obviously) not possible to acquire ctx->lock in order to read
- * both the above values in a consistent manner.
- */
- WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);
+ static void __update_context_guest_time(struct perf_event_context *ctx, bool adv)
+ {
+ lockdep_assert_held(&ctx->lock);
+
+ /* must be called after __update_context_time(); */
+ update_perf_time_ctx(&ctx->timeguest, ctx->time.stamp, adv);
}
static void update_context_time(struct perf_event_context *ctx)
{
__update_context_time(ctx, true);
+ if (is_guest_mediated_pmu_loaded())
+ __update_context_guest_time(ctx, true);
}
static u64 perf_event_time(struct perf_event *event)
if (is_cgroup_event(event))
return perf_cgroup_event_time(event);
- return ctx->time;
+ return __perf_event_time_ctx(event, &ctx->time);
}
static u64 perf_event_time_now(struct perf_event *event, u64 now)
return perf_cgroup_event_time_now(event, now);
if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
- return ctx->time;
+ return __perf_event_time_ctx(event, &ctx->time);
- now += READ_ONCE(ctx->timeoffset);
- return now;
+ return __perf_event_time_ctx_now(event, &ctx->time, now);
}
static enum event_type_t get_event_type(struct perf_event *event)
}
static inline void
- __ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx, bool final)
+ __ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx,
+ bool final, enum event_type_t event_type)
{
if (ctx->is_active & EVENT_TIME) {
if (ctx->is_active & EVENT_FROZEN)
return;
+
update_context_time(ctx);
- update_cgrp_time_from_cpuctx(cpuctx, final);
+ /* vPMU should not stop time */
+ update_cgrp_time_from_cpuctx(cpuctx, !(event_type & EVENT_GUEST) && final);
}
}
static inline void
ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx)
{
- __ctx_time_update(cpuctx, ctx, false);
+ __ctx_time_update(cpuctx, ctx, false, 0);
}
/*
static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
struct perf_event_context *ctx,
- struct pmu *pmu)
+ struct pmu *pmu,
+ enum event_type_t event_type)
{
- ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED);
+ ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED | event_type);
if (ctx)
- ctx_sched_in(ctx, pmu, EVENT_PINNED);
- ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);
+ ctx_sched_in(ctx, pmu, EVENT_PINNED | event_type);
+ ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE | event_type);
if (ctx)
- ctx_sched_in(ctx, pmu, EVENT_FLEXIBLE);
+ ctx_sched_in(ctx, pmu, EVENT_FLEXIBLE | event_type);
}
/*
event_type &= EVENT_ALL;
- for_each_epc(epc, &cpuctx->ctx, pmu, false)
+ for_each_epc(epc, &cpuctx->ctx, pmu, 0)
perf_pmu_disable(epc->pmu);
if (task_ctx) {
- for_each_epc(epc, task_ctx, pmu, false)
+ for_each_epc(epc, task_ctx, pmu, 0)
perf_pmu_disable(epc->pmu);
task_ctx_sched_out(task_ctx, pmu, event_type);
else if (event_type & EVENT_PINNED)
ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE);
- perf_event_sched_in(cpuctx, task_ctx, pmu);
+ perf_event_sched_in(cpuctx, task_ctx, pmu, 0);
- for_each_epc(epc, &cpuctx->ctx, pmu, false)
+ for_each_epc(epc, &cpuctx->ctx, pmu, 0)
perf_pmu_enable(epc->pmu);
if (task_ctx) {
- for_each_epc(epc, task_ctx, pmu, false)
+ for_each_epc(epc, task_ctx, pmu, 0)
perf_pmu_enable(epc->pmu);
}
}
ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type)
{
struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
+ enum event_type_t active_type = event_type & ~EVENT_FLAGS;
struct perf_event_pmu_context *pmu_ctx;
int is_active = ctx->is_active;
- bool cgroup = event_type & EVENT_CGROUP;
- event_type &= ~EVENT_CGROUP;
lockdep_assert_held(&ctx->lock);
*
* would only update time for the pinned events.
*/
- __ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx);
+ __ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx, event_type);
/*
* CPU-release for the below ->is_active store,
* see __load_acquire() in perf_event_time_now()
*/
barrier();
- ctx->is_active &= ~event_type;
+ ctx->is_active &= ~active_type;
if (!(ctx->is_active & EVENT_ALL)) {
/*
cpuctx->task_ctx = NULL;
}
- is_active ^= ctx->is_active; /* changed bits */
+ if (event_type & EVENT_GUEST) {
+ /*
+ * Schedule out all exclude_guest events of PMU
+ * with PERF_PMU_CAP_MEDIATED_VPMU.
+ */
+ is_active = EVENT_ALL;
+ __update_context_guest_time(ctx, false);
+ perf_cgroup_set_timestamp(cpuctx, true);
+ barrier();
+ } else {
+ is_active ^= ctx->is_active; /* changed bits */
+ }
- for_each_epc(pmu_ctx, ctx, pmu, cgroup)
+ for_each_epc(pmu_ctx, ctx, pmu, event_type)
__pmu_ctx_sched_out(pmu_ctx, is_active);
}
raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
if (context_equiv(ctx, next_ctx)) {
- perf_ctx_disable(ctx, false);
+ perf_ctx_disable(ctx, 0);
/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */
if (local_read(&ctx->nr_no_switch_fast) ||
perf_ctx_sched_task_cb(ctx, task, false);
- perf_ctx_enable(ctx, false);
+ perf_ctx_enable(ctx, 0);
/*
* RCU_INIT_POINTER here is safe because we've not
if (do_switch) {
raw_spin_lock(&ctx->lock);
- perf_ctx_disable(ctx, false);
+ perf_ctx_disable(ctx, 0);
inside_switch:
perf_ctx_sched_task_cb(ctx, task, false);
task_ctx_sched_out(ctx, NULL, EVENT_ALL);
- perf_ctx_enable(ctx, false);
+ perf_ctx_enable(ctx, 0);
raw_spin_unlock(&ctx->lock);
}
}
event_update_userpage(event);
}
+ struct merge_sched_data {
+ int can_add_hw;
+ enum event_type_t event_type;
+ };
+
static int merge_sched_in(struct perf_event *event, void *data)
{
struct perf_event_context *ctx = event->ctx;
- int *can_add_hw = data;
+ struct merge_sched_data *msd = data;
if (event->state <= PERF_EVENT_STATE_OFF)
return 0;
if (!event_filter_match(event))
return 0;
- if (group_can_go_on(event, *can_add_hw)) {
+ /*
+ * Don't schedule in any host events from PMU with
+ * PERF_PMU_CAP_MEDIATED_VPMU, while a guest is running.
+ */
+ if (is_guest_mediated_pmu_loaded() &&
+ event->pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU &&
+ !(msd->event_type & EVENT_GUEST))
+ return 0;
+
+ if (group_can_go_on(event, msd->can_add_hw)) {
if (!group_sched_in(event, ctx))
list_add_tail(&event->active_list, get_event_list(event));
}
if (event->state == PERF_EVENT_STATE_INACTIVE) {
- *can_add_hw = 0;
+ msd->can_add_hw = 0;
if (event->attr.pinned) {
perf_cgroup_event_disable(event, ctx);
perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
static void pmu_groups_sched_in(struct perf_event_context *ctx,
struct perf_event_groups *groups,
- struct pmu *pmu)
+ struct pmu *pmu,
+ enum event_type_t event_type)
{
- int can_add_hw = 1;
+ struct merge_sched_data msd = {
+ .can_add_hw = 1,
+ .event_type = event_type,
+ };
visit_groups_merge(ctx, groups, smp_processor_id(), pmu,
- merge_sched_in, &can_add_hw);
+ merge_sched_in, &msd);
}
static void __pmu_ctx_sched_in(struct perf_event_pmu_context *pmu_ctx,
struct perf_event_context *ctx = pmu_ctx->ctx;
if (event_type & EVENT_PINNED)
- pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu);
+ pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu, event_type);
if (event_type & EVENT_FLEXIBLE)
- pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu);
+ pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu, event_type);
}
static void
ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type)
{
struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
+ enum event_type_t active_type = event_type & ~EVENT_FLAGS;
struct perf_event_pmu_context *pmu_ctx;
int is_active = ctx->is_active;
- bool cgroup = event_type & EVENT_CGROUP;
-
- event_type &= ~EVENT_CGROUP;
lockdep_assert_held(&ctx->lock);
return;
if (!(is_active & EVENT_TIME)) {
+ /* EVENT_TIME should be active while the guest runs */
+ WARN_ON_ONCE(event_type & EVENT_GUEST);
/* start ctx time */
__update_context_time(ctx, false);
- perf_cgroup_set_timestamp(cpuctx);
+ perf_cgroup_set_timestamp(cpuctx, false);
/*
* CPU-release for the below ->is_active store,
* see __load_acquire() in perf_event_time_now()
barrier();
}
- ctx->is_active |= (event_type | EVENT_TIME);
+ ctx->is_active |= active_type | EVENT_TIME;
if (ctx->task) {
if (!(is_active & EVENT_ALL))
cpuctx->task_ctx = ctx;
WARN_ON_ONCE(cpuctx->task_ctx != ctx);
}
- is_active ^= ctx->is_active; /* changed bits */
+ if (event_type & EVENT_GUEST) {
+ /*
+ * Schedule in the required exclude_guest events of PMU
+ * with PERF_PMU_CAP_MEDIATED_VPMU.
+ */
+ is_active = event_type & EVENT_ALL;
+
+ /*
+ * Update ctx time to set the new start time for
+ * the exclude_guest events.
+ */
+ update_context_time(ctx);
+ update_cgrp_time_from_cpuctx(cpuctx, false);
+ barrier();
+ } else {
+ is_active ^= ctx->is_active; /* changed bits */
+ }
/*
* First go through the list and put on any pinned groups
* in order to give them the best chance of going on.
*/
if (is_active & EVENT_PINNED) {
- for_each_epc(pmu_ctx, ctx, pmu, cgroup)
- __pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED);
+ for_each_epc(pmu_ctx, ctx, pmu, event_type)
+ __pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED | (event_type & EVENT_GUEST));
}
/* Then walk through the lower prio flexible groups */
if (is_active & EVENT_FLEXIBLE) {
- for_each_epc(pmu_ctx, ctx, pmu, cgroup)
- __pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE);
+ for_each_epc(pmu_ctx, ctx, pmu, event_type)
+ __pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE | (event_type & EVENT_GUEST));
}
}
if (cpuctx->task_ctx == ctx) {
perf_ctx_lock(cpuctx, ctx);
- perf_ctx_disable(ctx, false);
+ perf_ctx_disable(ctx, 0);
perf_ctx_sched_task_cb(ctx, task, true);
- perf_ctx_enable(ctx, false);
+ perf_ctx_enable(ctx, 0);
perf_ctx_unlock(cpuctx, ctx);
goto rcu_unlock;
}
if (!ctx->nr_events)
goto unlock;
- perf_ctx_disable(ctx, false);
+ perf_ctx_disable(ctx, 0);
/*
* We want to keep the following priority order:
* cpu pinned (that don't need to move), task pinned,
* events, no need to flip the cpuctx's events around.
*/
if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {
- perf_ctx_disable(&cpuctx->ctx, false);
+ perf_ctx_disable(&cpuctx->ctx, 0);
ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE);
}
- perf_event_sched_in(cpuctx, ctx, NULL);
+ perf_event_sched_in(cpuctx, ctx, NULL, 0);
perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true);
if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
- perf_ctx_enable(&cpuctx->ctx, false);
+ perf_ctx_enable(&cpuctx->ctx, 0);
- perf_ctx_enable(ctx, false);
+ perf_ctx_enable(ctx, 0);
unlock:
perf_ctx_unlock(cpuctx, ctx);
{
struct pmu *pmu = event->pmu;
+ security_perf_event_free(event);
+
if (event->attach_state & PERF_ATTACH_CALLCHAIN)
put_callchain_buffers();
call_rcu(&event->rcu_head, free_event_rcu);
}
+ static void mediated_pmu_unaccount_event(struct perf_event *event);
+
DEFINE_FREE(__free_event, struct perf_event *, if (_T) __free_event(_T))
/* vs perf_event_alloc() success */
irq_work_sync(&event->pending_disable_irq);
unaccount_event(event);
-
- security_perf_event_free(event);
+ mediated_pmu_unaccount_event(event);
if (event->rb) {
/*
}
EXPORT_SYMBOL_GPL(perf_event_pause);
+ #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
+ static atomic_t nr_include_guest_events __read_mostly;
+
+ static atomic_t nr_mediated_pmu_vms __read_mostly;
+ static DEFINE_MUTEX(perf_mediated_pmu_mutex);
+
+ /* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */
+ static inline bool is_include_guest_event(struct perf_event *event)
+ {
+ if ((event->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) &&
+ !event->attr.exclude_guest)
+ return true;
+
+ return false;
+ }
+
+ static int mediated_pmu_account_event(struct perf_event *event)
+ {
+ if (!is_include_guest_event(event))
+ return 0;
+
+ if (atomic_inc_not_zero(&nr_include_guest_events))
+ return 0;
+
+ guard(mutex)(&perf_mediated_pmu_mutex);
+ if (atomic_read(&nr_mediated_pmu_vms))
+ return -EOPNOTSUPP;
+
+ atomic_inc(&nr_include_guest_events);
+ return 0;
+ }
+
+ static void mediated_pmu_unaccount_event(struct perf_event *event)
+ {
+ if (!is_include_guest_event(event))
+ return;
+
+ if (WARN_ON_ONCE(!atomic_read(&nr_include_guest_events)))
+ return;
+
+ atomic_dec(&nr_include_guest_events);
+ }
+
+ /*
+ * Currently invoked at VM creation to
+ * - Check whether there are existing !exclude_guest events of PMU with
+ * PERF_PMU_CAP_MEDIATED_VPMU
+ * - Set nr_mediated_pmu_vms to prevent !exclude_guest event creation on
+ * PMUs with PERF_PMU_CAP_MEDIATED_VPMU
+ *
+ * No impact for the PMU without PERF_PMU_CAP_MEDIATED_VPMU. The perf
+ * still owns all the PMU resources.
+ */
+ int perf_create_mediated_pmu(void)
+ {
+ if (atomic_inc_not_zero(&nr_mediated_pmu_vms))
+ return 0;
+
+ guard(mutex)(&perf_mediated_pmu_mutex);
+ if (atomic_read(&nr_include_guest_events))
+ return -EBUSY;
+
+ atomic_inc(&nr_mediated_pmu_vms);
+ return 0;
+ }
+ EXPORT_SYMBOL_FOR_KVM(perf_create_mediated_pmu);
+
+ void perf_release_mediated_pmu(void)
+ {
+ if (WARN_ON_ONCE(!atomic_read(&nr_mediated_pmu_vms)))
+ return;
+
+ atomic_dec(&nr_mediated_pmu_vms);
+ }
+ EXPORT_SYMBOL_FOR_KVM(perf_release_mediated_pmu);
+
+ /* When loading a guest's mediated PMU, schedule out all exclude_guest events. */
+ void perf_load_guest_context(void)
+ {
+ struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
+
+ lockdep_assert_irqs_disabled();
+
+ guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
+
+ if (WARN_ON_ONCE(__this_cpu_read(guest_ctx_loaded)))
+ return;
+
+ perf_ctx_disable(&cpuctx->ctx, EVENT_GUEST);
+ ctx_sched_out(&cpuctx->ctx, NULL, EVENT_GUEST);
+ if (cpuctx->task_ctx) {
+ perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST);
+ task_ctx_sched_out(cpuctx->task_ctx, NULL, EVENT_GUEST);
+ }
+
+ perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST);
+ if (cpuctx->task_ctx)
+ perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST);
+
+ __this_cpu_write(guest_ctx_loaded, true);
+ }
+ EXPORT_SYMBOL_GPL(perf_load_guest_context);
+
+ void perf_put_guest_context(void)
+ {
+ struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
+
+ lockdep_assert_irqs_disabled();
+
+ guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx);
+
+ if (WARN_ON_ONCE(!__this_cpu_read(guest_ctx_loaded)))
+ return;
+
+ perf_ctx_disable(&cpuctx->ctx, EVENT_GUEST);
+ if (cpuctx->task_ctx)
+ perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST);
+
+ perf_event_sched_in(cpuctx, cpuctx->task_ctx, NULL, EVENT_GUEST);
+
+ if (cpuctx->task_ctx)
+ perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST);
+ perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST);
+
+ __this_cpu_write(guest_ctx_loaded, false);
+ }
+ EXPORT_SYMBOL_GPL(perf_put_guest_context);
+ #else
+ static int mediated_pmu_account_event(struct perf_event *event) { return 0; }
+ static void mediated_pmu_unaccount_event(struct perf_event *event) {}
+ #endif
+
/*
* Holding the top-level event's child_mutex means that any
* descendant process that has inherited this event will block
goto unlock;
/*
- * compute total_time_enabled, total_time_running
- * based on snapshot values taken when the event
- * was last scheduled in.
+ * Disable preemption to guarantee consistent time stamps are stored to
+ * the user page.
+ */
+ preempt_disable();
+
+ /*
+ * Compute total_time_enabled, total_time_running based on snapshot
+ * values taken when the event was last scheduled in.
*
- * we cannot simply called update_context_time()
- * because of locking issue as we can be called in
- * NMI context
+ * We cannot simply call update_context_time() because doing so would
+ * lead to deadlock when called from NMI context.
*/
calc_timer_values(event, &now, &enabled, &running);
userpg = rb->user_page;
- /*
- * Disable preemption to guarantee consistent time stamps are stored to
- * the user page.
- */
- preempt_disable();
+
++userpg->lock;
barrier();
userpg->index = perf_event_index(event);
if (data_page_nr(event->rb) != nr_pages)
return -EINVAL;
+ /*
+ * If this event doesn't have mmap_count, we're attempting to
+ * create an alias of another event's mmap(); this would mean
+ * both events will end up scribbling the same user_page;
+ * which makes no sense.
+ */
+ if (!refcount_read(&event->mmap_count))
+ return -EBUSY;
+
if (refcount_inc_not_zero(&event->rb->mmap_count)) {
/*
* Success -- managed to mmap() the same buffer
DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
+ DEFINE_STATIC_CALL_RET0(__perf_guest_handle_mediated_pmi, *perf_guest_cbs->handle_mediated_pmi);
void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
if (cbs->handle_intel_pt_intr)
static_call_update(__perf_guest_handle_intel_pt_intr,
cbs->handle_intel_pt_intr);
+
+ if (cbs->handle_mediated_pmi)
+ static_call_update(__perf_guest_handle_mediated_pmi,
+ cbs->handle_mediated_pmi);
}
EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
rcu_assign_pointer(perf_guest_cbs, NULL);
static_call_update(__perf_guest_state, (void *)&__static_call_return0);
static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0);
- static_call_update(__perf_guest_handle_intel_pt_intr,
- (void *)&__static_call_return0);
+ static_call_update(__perf_guest_handle_intel_pt_intr, (void *)&__static_call_return0);
+ static_call_update(__perf_guest_handle_mediated_pmi, (void *)&__static_call_return0);
synchronize_rcu();
}
EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
if (user_mode(regs)) {
regs_user->abi = perf_reg_abi(current);
regs_user->regs = regs;
- } else if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER))) {
+ } else if (is_user_task(current)) {
perf_get_regs_user(regs_user, regs);
} else {
regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
u64 read_format = event->attr.read_format;
/*
- * compute total_time_enabled, total_time_running
- * based on snapshot values taken when the event
- * was last scheduled in.
+ * Compute total_time_enabled, total_time_running based on snapshot
+ * values taken when the event was last scheduled in.
*
- * we cannot simply called update_context_time()
- * because of locking issue as we are called in
- * NMI context
+ * We cannot simply call update_context_time() because doing so would
+ * lead to deadlock when called from NMI context.
*/
if (read_format & PERF_FORMAT_TOTAL_TIMES)
calc_timer_values(event, &now, &enabled, &running);
* Try IRQ-safe get_user_page_fast_only first.
* If failed, leave phys_addr as 0.
*/
- if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER))) {
+ if (is_user_task(current)) {
struct page *p;
pagefault_disable();
{
bool kernel = !event->attr.exclude_callchain_kernel;
bool user = !event->attr.exclude_callchain_user &&
- !(current->flags & (PF_KTHREAD | PF_USER_WORKER));
+ is_user_task(current);
/* Disallow cross-task user callchains. */
bool crosstask = event->ctx->task && event->ctx->task != current;
bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) && user &&
}
}
+static void perf_swevent_destroy_hrtimer(struct perf_event *event)
+{
+ hrtimer_cancel(&event->hw.hrtimer);
+}
+
static void perf_swevent_init_hrtimer(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
return;
hrtimer_setup(&hwc->hrtimer, perf_swevent_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
+ event->destroy = perf_swevent_destroy_hrtimer;
/*
* Since hrtimers have a fixed rate, we can do a static freq->period
static void task_clock_event_start(struct perf_event *event, int flags)
{
event->hw.state = 0;
- local64_set(&event->hw.prev_count, event->ctx->time);
+ local64_set(&event->hw.prev_count, event->ctx->time.time);
perf_swevent_start_hrtimer(event);
}
event->hw.state = PERF_HES_STOPPED;
perf_swevent_cancel_hrtimer(event);
if (flags & PERF_EF_UPDATE)
- task_clock_event_update(event, event->ctx->time);
+ task_clock_event_update(event, event->ctx->time.time);
}
static int task_clock_event_add(struct perf_event *event, int flags)
static void task_clock_event_read(struct perf_event *event)
{
u64 now = perf_clock();
- u64 delta = now - event->ctx->timestamp;
- u64 time = event->ctx->time + delta;
+ u64 delta = now - event->ctx->time.stamp;
+ u64 time = event->ctx->time.time + delta;
task_clock_event_update(event, time);
}
if (err)
return ERR_PTR(err);
+ err = mediated_pmu_account_event(event);
+ if (err)
+ return ERR_PTR(err);
+
/* symmetric to unaccount_event() in _free_event() */
account_event(event);
kvm_free_memslot(kvm, old);
break;
case KVM_MR_MOVE:
+ /*
+ * Moving a guest_memfd memslot isn't supported, and will never
+ * be supported.
+ */
+ WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD);
+ fallthrough;
case KVM_MR_FLAGS_ONLY:
/*
* Free the dirty bitmap as needed; the below check encompasses
if (old->dirty_bitmap && !new->dirty_bitmap)
kvm_destroy_dirty_bitmap(old);
+ /*
+ * Unbind the guest_memfd instance as needed; the @new slot has
+ * already created its own binding. TODO: Drop the WARN when
+ * dirty logging guest_memfd memslots is supported. Until then,
+ * flags-only changes on guest_memfd slots should be impossible.
+ */
+ if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD))
+ kvm_gmem_unbind(old);
+
/*
* The final quirk. Free the detached, old slot, but only its
* memory, not any metadata. Metadata, including arch specific
return -EINVAL;
if ((mem->userspace_addr != old->userspace_addr) ||
(npages != old->npages) ||
- ((mem->flags ^ old->flags) & KVM_MEM_READONLY))
+ ((mem->flags ^ old->flags) & (KVM_MEM_READONLY | KVM_MEM_GUEST_MEMFD)))
return -EINVAL;
if (base_gfn != old->base_gfn)
bool writable)
{
struct kvm_follow_pfn kfp = {
- .slot = gfn_to_memslot(vcpu->kvm, gfn),
+ .slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn),
.gfn = gfn,
.flags = writable ? FOLL_WRITE : 0,
.refcounted_page = &map->pinned_page,
.state = kvm_guest_state,
.get_ip = kvm_guest_get_ip,
.handle_intel_pt_intr = NULL,
+ .handle_mediated_pmi = NULL,
};
- void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void))
+ void __kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void),
+ void (*mediated_pmi_handler)(void))
{
kvm_guest_cbs.handle_intel_pt_intr = pt_intr_handler;
+ kvm_guest_cbs.handle_mediated_pmi = mediated_pmi_handler;
+
perf_register_guest_info_callbacks(&kvm_guest_cbs);
}
void kvm_unregister_perf_callbacks(void)
projects / linux.git / commitdiff