path: root/arch/powerpc/sysdev/xive/common.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright 2016,2017 IBM Corporation.
 */

#define pr_fmt(fmt) "xive: " fmt

#include <linux/types.h>
#include <linux/threads.h>
#include <linux/kernel.h>
#include <linux/irq.h>
#include <linux/debugfs.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/seq_file.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/msi.h>
#include <linux/vmalloc.h>

#include <asm/prom.h>
#include <asm/io.h>
#include <asm/smp.h>
#include <asm/machdep.h>
#include <asm/irq.h>
#include <asm/errno.h>
#include <asm/xive.h>
#include <asm/xive-regs.h>
#include <asm/xmon.h>

#include "xive-internal.h"

#undef DEBUG_FLUSH
#undef DEBUG_ALL

#ifdef DEBUG_ALL
#define DBG_VERBOSE(fmt, ...)	pr_devel("cpu %d - " fmt, \
					 smp_processor_id(), ## __VA_ARGS__)
#else
#define DBG_VERBOSE(fmt...)	do { } while(0)
#endif

bool __xive_enabled;
EXPORT_SYMBOL_GPL(__xive_enabled);
bool xive_cmdline_disabled;

/* We use only one priority for now */
static u8 xive_irq_priority;

/* TIMA exported to KVM */
void __iomem *xive_tima;
EXPORT_SYMBOL_GPL(xive_tima);
u32 xive_tima_offset;

/* Backend ops */
static const struct xive_ops *xive_ops;

/* Our global interrupt domain */
static struct irq_domain *xive_irq_domain;

#ifdef CONFIG_SMP
/* The IPIs all use the same logical irq number */
static u32 xive_ipi_irq;
#endif

/* Xive state for each CPU */
static DEFINE_PER_CPU(struct xive_cpu *, xive_cpu);

/* An invalid CPU target */
#define XIVE_INVALID_TARGET	(-1)

/*
 * Read the next entry in a queue, return its content if it's valid
 * or 0 if there is no new entry.
 *
 * The queue pointer is moved forward unless "just_peek" is set
 */
static u32 xive_read_eq(struct xive_q *q, bool just_peek)
{
	u32 cur;

	if (!q->qpage)
		return 0;
	cur = be32_to_cpup(q->qpage + q->idx);

	/* Check valid bit (31) vs current toggle polarity */
	if ((cur >> 31) == q->toggle)
		return 0;

	/* If consuming from the queue ... */
	if (!just_peek) {
		/* Next entry */
		q->idx = (q->idx + 1) & q->msk;

		/* Wrap around: flip valid toggle */
		if (q->idx == 0)
			q->toggle ^= 1;
	}
	/* Mask out the valid bit (31) */
	return cur & 0x7fffffff;
}

/*
 * Scans all the queue that may have interrupts in them
 * (based on "pending_prio") in priority order until an
 * interrupt is found or all the queues are empty.
 *
 * Then updates the CPPR (Current Processor Priority
 * Register) based on the most favored interrupt found
 * (0xff if none) and return what was found (0 if none).
 *
 * If just_peek is set, return the most favored pending
 * interrupt if any but don't update the queue pointers.
 *
 * Note: This function can operate generically on any number
 * of queues (up to 8). The current implementation of the XIVE
 * driver only uses a single queue however.
 *
 * Note2: This will also "flush" "the pending_count" of a queue
 * into the "count" when that queue is observed to be empty.
 * This is used to keep track of the amount of interrupts
 * targetting a queue. When an interrupt is moved away from
 * a queue, we only decrement that queue count once the queue
 * has been observed empty to avoid races.
 */
static u32 xive_scan_interrupts(struct xive_cpu *xc, bool just_peek)
{
	u32 irq = 0;
	u8 prio = 0;

	/* Find highest pending priority */
	while (xc->pending_prio != 0) {
		struct xive_q *q;

		prio = ffs(xc->pending_prio) - 1;
		DBG_VERBOSE("scan_irq: trying prio %d\n", prio);

		/* Try to fetch */
		irq = xive_read_eq(&xc->queue[prio], just_peek);

		/* Found something ? That's it */
		if (irq) {
			if (just_peek || irq_to_desc(irq))
				break;
			/*
			 * We should never get here; if we do then we must
			 * have failed to synchronize the interrupt properly
			 * when shutting it down.
			 */
			pr_crit("xive: got interrupt %d without descriptor, dropping\n",
				irq);
			WARN_ON(1);
			continue;
		}

		/* Clear pending bits */
		xc->pending_prio &= ~(1 << prio);

		/*
		 * Check if the queue count needs adjusting due to
		 * interrupts being moved away. See description of
		 * xive_dec_target_count()
		 */
		q = &xc->queue[prio];
		if (atomic_read(&q->pending_count)) {
			int p = atomic_xchg(&q->pending_count, 0);
			if (p) {
				WARN_ON(p > atomic_read(&q->count));
				atomic_sub(p, &q->count);
			}
		}
	}

	/* If nothing was found, set CPPR to 0xff */
	if (irq == 0)
		prio = 0xff;

	/* Update HW CPPR to match if necessary */
	if (prio != xc->cppr) {
		DBG_VERBOSE("scan_irq: adjusting CPPR to %d\n", prio);
		xc->cppr = prio;
		out_8(xive_tima + xive_tima_offset + TM_CPPR, prio);
	}

	return irq;
}

/*
 * This is used to perform the magic loads from an ESB
 * described in xive-regs.h
 */
static notrace u8 xive_esb_read(struct xive_irq_data *xd, u32 offset)
{
	u64 val;

	/* Handle HW errata */
	if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)
		offset |= offset << 4;

	if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw)
		val = xive_ops->esb_rw(xd->hw_irq, offset, 0, 0);
	else
		val = in_be64(xd->eoi_mmio + offset);

	return (u8)val;
}

static void xive_esb_write(struct xive_irq_data *xd, u32 offset, u64 data)
{
	/* Handle HW errata */
	if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)
		offset |= offset << 4;

	if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw)
		xive_ops->esb_rw(xd->hw_irq, offset, data, 1);
	else
		out_be64(xd->eoi_mmio + offset, data);
}

#ifdef CONFIG_XMON
static notrace void xive_dump_eq(const char *name, struct xive_q *q)
{
	u32 i0, i1, idx;

	if (!q->qpage)
		return;
	idx = q->idx;
	i0 = be32_to_cpup(q->qpage + idx);
	idx = (idx + 1) & q->msk;
	i1 = be32_to_cpup(q-