path: root/arch/powerpc/kvm/book3s_xics.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2012 Michael Ellerman, IBM Corporation.
 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation.
 */

#include <linux/kernel.h>
#include <linux/kvm_host.h>
#include <linux/err.h>
#include <linux/gfp.h>
#include <linux/anon_inodes.h>
#include <linux/spinlock.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>

#include <asm/kvm_book3s.h>
#include <asm/kvm_ppc.h>
#include <asm/hvcall.h>
#include <asm/xics.h>
#include <asm/time.h>

#include <linux/seq_file.h>

#include "book3s_xics.h"

#if 1
#define XICS_DBG(fmt...) do { } while (0)
#else
#define XICS_DBG(fmt...) trace_printk(fmt)
#endif

#define ENABLE_REALMODE	true
#define DEBUG_REALMODE	false

/*
 * LOCKING
 * =======
 *
 * Each ICS has a spin lock protecting the information about the IRQ
 * sources and avoiding simultaneous deliveries of the same interrupt.
 *
 * ICP operations are done via a single compare & swap transaction
 * (most ICP state fits in the union kvmppc_icp_state)
 */

/*
 * TODO
 * ====
 *
 * - To speed up resends, keep a bitmap of "resend" set bits in the
 *   ICS
 *
 * - Speed up server# -> ICP lookup (array ? hash table ?)
 *
 * - Make ICS lockless as well, or at least a per-interrupt lock or hashed
 *   locks array to improve scalability
 */

/* -- ICS routines -- */

static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
			    u32 new_irq, bool check_resend);

/*
 * Return value ideally indicates how the interrupt was handled, but no
 * callers look at it (given that we don't implement KVM_IRQ_LINE_STATUS),
 * so just return 0.
 */
static int ics_deliver_irq(struct kvmppc_xics *xics, u32 irq, u32 level)
{
	struct ics_irq_state *state;
	struct kvmppc_ics *ics;
	u16 src;
	u32 pq_old, pq_new;

	XICS_DBG("ics deliver %#x (level: %d)\n", irq, level);

	ics = kvmppc_xics_find_ics(xics, irq, &src);
	if (!ics) {
		XICS_DBG("ics_deliver_irq: IRQ 0x%06x not found !\n", irq);
		return -EINVAL;
	}
	state = &ics->irq_state[src];
	if (!state->exists)
		return -EINVAL;

	if (level == KVM_INTERRUPT_SET_LEVEL || level == KVM_INTERRUPT_SET)
		level = 1;
	else if (level == KVM_INTERRUPT_UNSET)
		level = 0;
	/*
	 * Take other values the same as 1, consistent with original code.
	 * maybe WARN here?
	 */

	if (!state->lsi && level == 0) /* noop for MSI */
		return 0;

	do {
		pq_old = state->pq_state;
		if (state->lsi) {
			if (level) {
				if (pq_old & PQ_PRESENTED)
					/* Setting already set LSI ... */
					return 0;

				pq_new = PQ_PRESENTED;
			} else
				pq_new = 0;
		} else
			pq_new = ((pq_old << 1) & 3) | PQ_PRESENTED;
	} while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old);

	/* Test P=1, Q=0, this is the only case where we present */
	if (pq_new == PQ_PRESENTED)
		icp_deliver_irq(xics, NULL, irq, false);

	/* Record which CPU this arrived on for passed-through interrupts */
	if (state->host_irq)
		state->intr_cpu = raw_smp_processor_id();

	return 0;
}

static void ics_check_resend(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
			     struct kvmppc_icp *icp)
{
	int i;

	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
		struct ics_irq_state *state = &ics->irq_state[i];
		if (state->resend) {
			XICS_DBG("resend %#x prio %#x\n", state->number,
				      state->priority);
			icp_deliver_irq(xics, icp, state->number, true);
		}
	}
}

static bool write_xive(struct kvmppc_xics *xics, struct kvmppc_ics *ics,
		       struct ics_irq_state *state,
		       u32 server, u32 priority, u32 saved_priority)
{
	bool deliver;
	unsigned long flags;

	local_irq_save(flags);
	arch_spin_lock(&ics->lock);

	state->server = server;
	state->priority = priority;
	state->saved_priority = saved_priority;
	deliver = false;
	if ((state->masked_pending || state->resend) && priority != MASKED) {
		state->masked_pending = 0;
		state->resend = 0;
		deliver = true;
	}

	arch_spin_unlock(&ics->lock);
	local_irq_restore(flags);

	return deliver;
}

int kvmppc_xics_set_xive(struct kvm *kvm, u32 irq, u32 server, u32 priority)
{
	struct kvmppc_xics *xics = kvm->arch.xics;
	struct kvmppc_icp *icp;
	struct kvmppc_ics *ics;
	struct ics_irq_state *state;
	u16 src;

	if (!xics)
		return -ENODEV;

	ics = kvmppc_xics_find_ics(xics, irq, &src);
	if (!ics)
		return -EINVAL;
	state = &ics->irq_state[src];

	icp = kvmppc_xics_find_server(kvm, server);
	if (!icp)
		return -EINVAL;

	XICS_DBG("set_xive %#x server %#x prio %#x MP:%d RS:%d\n",
		 irq, server, priority,
		 state->masked_pending, state->resend);

	if (write_xive(xics, ics, state, server, priority, priority))
		icp_deliver_irq(xics, icp, irq, false);

	return 0;
}

int kvmppc_xics_get_xive(struct kvm *kvm, u32 irq, u32 *server, u32 *priority)
{
	struct kvmppc_xics *xics = kvm->arch.xics;
	struct kvmppc_ics *ics;
	struct ics_irq_state *state;
	u16 src;
	unsigned long flags;

	if (!xics)
		return -ENODEV;

	ics = kvmppc_xics_find_ics(xics, irq, &src);
	if (!ics)
		return -EINVAL;
	state = &ics->irq_state[src];

	local_irq_save(flags);
	arch_spin_lock(&ics->lock);
	*server = state->server;
	*priority = state->priority;
	arch_spin_unlock(&ics->lock);
	local_irq_restore(flags);

	return 0;
}

int kvmppc_xics_int_on(struct kvm *kvm, u32 irq)
{
	struct kvmppc_xics *xics = kvm->arch.xics;
	struct kvmppc_icp *icp;
	struct kvmppc_ics *ics;
	struct ics_irq_state *state;
	u16 src;

	if (!xics)
		return -ENODEV;

	ics = kvmppc_xics_find_ics(xics, irq, &src);
	if (!ics)
		return -EINVAL;
	state = &ics->irq_state[src];

	icp = kvmppc_xics_find_server(kvm, state->server);
	if (!icp)
		return -EINVAL;

	if (write_xive(xics, ics, state, state->server, state->saved_priority,
		       state->saved_priority))
		icp_deliver_irq(xics, icp, irq, false);

	return 0;
}

int kvmppc_xics_int_off(struct kvm *kvm, u32 irq)
{
	struct kvmppc_xics *xics = kvm->arch.xics;
	struct kvmppc_ics *ics;
	struct ics_irq_state *state;
	u16 src;

	if (!xics)
		return -ENODEV;

	ics = kvmppc_xics_find_ics(xics, irq, &src);
	if (!ics)
		return -EINVAL;
	state = &ics->irq_state[src];

	write_xive(xics, ics, state, state->server, MASKED, state->priority);

	return 0;
}

/* -- ICP routines, including hcalls -- */

static inline bool icp_try_update(struct kvmppc_icp *icp,
				  union kvmppc_icp_state old,
				  union kvmppc_icp_state new,
				  bool change_self)
{
	bool success;

	/* Calculate new output value */
	new.out_ee = (new.xisr && (new.pending_pri < new.cppr));

	/* Attempt atomic update */
	success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw;
	if (!success)
		goto bail;

	XICS_DBG("UPD [%04lx] - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n",
		 icp->server_num,
		 old.cppr, old.mfrr, old.pending_pri, old.xisr,
		 old.need_resend, old.out_ee);
	XICS_DBG("UPD        - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n",
		 new.cppr, new.mfrr, new.pending_pri, new.xisr,
		 new.need_resend, new.out_ee);
	/*
	 * Check for output state update
	 *
	 * Note that this is racy since another processor could be updating
	 * the state already. This is why we never clear the interrupt output
	 * here, we only ever set it. The clear only happens prior to doing
	 * an update and only by the processor itself. Currently we do it
	 * in Accept (H_XIRR) and Up_Cppr (H_XPPR).
	 *
	 * We also do not try to figure out whether the EE state has changed,
	 * we unconditionally set it if the new state calls for it. The reason
	 * for that is that we opportunistically remove the pending interrupt
	 * flag when raising CPPR, so we need to set it back here if an
	 * interrupt is still pending.
	 */
	if (new.out_ee) {
		kvmppc_book3s_queue_irqprio(icp->vcpu,
					    BOOK3S_INTERRUPT_EXTERNAL);
		if (!change_self)
			kvmppc_fast_vcpu_kick(icp->vcpu);
	}
 bail:
	return success;
}

static void icp_check_resend(struct kvmppc_xics *xics,
			     struct kvmppc_icp *icp)
{
	u32 icsid;

	/* Order this load with the test for need_resend in the caller */
	smp_rmb();
	for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) {
		struct kvmppc_ics *ics = xics->ics[icsid];

		if (!test_and_clear_bit(icsid, icp->resend_map))
			continue;
		if (!ics)
			continue;
		ics_check_resend(xics, ics, icp);
	}
}

static bool icp_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority,
			       u32 *reject)
{
	union kvmppc_icp_state old_state, new_state;
	bool succe