path: root/arch/s390/kernel/perf_pai_ext.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Performance event support - Processor Activity Instrumentation Extension
 * Facility
 *
 *  Copyright IBM Corp. 2022
 *  Author(s): Thomas Richter <tmricht@linux.ibm.com>
 */
#define KMSG_COMPONENT	"pai_ext"
#define pr_fmt(fmt)	KMSG_COMPONENT ": " fmt

#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/io.h>
#include <linux/perf_event.h>
#include <asm/ctlreg.h>
#include <asm/pai.h>
#include <asm/debug.h>

#define	PAIE1_CB_SZ		0x200	/* Size of PAIE1 control block */
#define	PAIE1_CTRBLOCK_SZ	0x400	/* Size of PAIE1 counter blocks */

static debug_info_t *paiext_dbg;
static unsigned int paiext_cnt;	/* Extracted with QPACI instruction */

struct pai_userdata {
	u16 num;
	u64 value;
} __packed;

/* Create the PAI extension 1 control block area.
 * The PAI extension control block 1 is pointed to by lowcore
 * address 0x1508 for each CPU. This control block is 512 bytes in size
 * and requires a 512 byte boundary alignment.
 */
struct paiext_cb {		/* PAI extension 1 control block */
	u64 header;		/* Not used */
	u64 reserved1;
	u64 acc;		/* Addr to analytics counter control block */
	u8 reserved2[488];
} __packed;

struct paiext_map {
	unsigned long *area;		/* Area for CPU to store counters */
	struct pai_userdata *save;	/* Area to store non-zero counters */
	enum paievt_mode mode;		/* Type of event */
	unsigned int active_events;	/* # of PAI Extension users */
	refcount_t refcnt;
	struct perf_event *event;	/* Perf event for sampling */
	struct paiext_cb *paiext_cb;	/* PAI extension control block area */
};

struct paiext_mapptr {
	struct paiext_map *mapptr;
};

static struct paiext_root {		/* Anchor to per CPU data */
	refcount_t refcnt;		/* Overall active events */
	struct paiext_mapptr __percpu *mapptr;
} paiext_root;

/* Free per CPU data when the last event is removed. */
static void paiext_root_free(void)
{
	if (refcount_dec_and_test(&paiext_root.refcnt)) {
		free_percpu(paiext_root.mapptr);
		paiext_root.mapptr = NULL;
	}
}

/* On initialization of first event also allocate per CPU data dynamically.
 * Start with an array of pointers, the array size is the maximum number of
 * CPUs possible, which might be larger than the number of CPUs currently
 * online.
 */
static int paiext_root_alloc(void)
{
	if (!refcount_inc_not_zero(&paiext_root.refcnt)) {
		/* The memory is already zeroed. */
		paiext_root.mapptr = alloc_percpu(struct paiext_mapptr);
		if (!paiext_root.mapptr) {
			/* Returning without refcnt adjustment is ok. The
			 * error code is handled by paiext_alloc() which
			 * decrements refcnt when an event can not be
			 * created.
			 */
			return -ENOMEM;
		}
		refcount_set(&paiext_root.refcnt, 1);
	}
	return 0;
}

/* Protects against concurrent increment of sampler and counter member
 * increments at the same time and prohibits concurrent execution of
 * counting and sampling events.
 * Ensures that analytics counter block is deallocated only when the
 * sampling and counting on that cpu is zero.
 * For details see paiext_alloc().
 */
static DEFINE_MUTEX(paiext_reserve_mutex);

/* Free all memory allocated for event counting/sampling setup */
static void paiext_free(struct paiext_mapptr *mp)
{
	kfree(mp->mapptr->area);
	kfree(mp->mapptr->paiext_cb);
	kvfree(mp->mapptr->save);
	kfree(mp->mapptr);
	mp->mapptr = NULL;
}

/* Release the PMU if event is the last perf event */
static void paiext_event_destroy(struct perf_event *event)
{
	struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, event->cpu);
	struct paiext_map *cpump = mp->mapptr;

	mutex_lock(&paiext_reserve_mutex);
	cpump->event = NULL;
	if (refcount_dec_and_test(&cpump->refcnt))	/* Last reference gone */
		paiext_free(mp);
	paiext_root_free();
	mutex_unlock(&paiext_reserve_mutex);
	debug_sprintf_event(paiext_dbg, 4, "%s cpu %d mapptr %p\n", __func__,
			    event->cpu, mp->mapptr);

}

/* Used to avoid races in checking concurrent access of counting and
 * sampling for pai_extension events.
 *
 * Only one instance of event pai_ext/NNPA_ALL/ for sampling is
 * allowed and when this event is running, no counting event is allowed.
 * Several counting events are allowed in parallel, but no sampling event
 * is allowed while one (or more) counting events are running.
 *
 * This function is called in process context and it is safe to block.
 * When the event initialization functions fails, no other call back will
 * be invoked.
 *
 * Allocate the memory for the event.
 */
static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event)
{
	struct paiext_mapptr *mp;
	struct paiext_map *cpump;
	int rc;

	mutex_lock(&paiext_reserve_mutex);

	rc = paiext_root_alloc();
	if (rc)
		goto unlock;

	mp = per_cpu_ptr(paiext_root.mapptr, event->cpu);
	cpump = mp->mapptr;
	if (!cpump) {			/* Paiext_map allocated? */
		rc = -ENOMEM;
		cpump = kzalloc(sizeof(*cpump), GFP_KERNEL);
		if (!cpump)
			goto undo;

		/* Allocate memory for counter area and counter extraction.
		 * These are
		 * - a 512 byte block and requires 512 byte boundary alignment.
		 * - a 1KB byte block and requires 1KB boundary alignment.
		 * Only the first counting event has to allocate the area.
		 *
		 * Note: This works with commit 59bb47985c1d by default.
		 * Backporting this to kernels without this commit might
		 * need adjustment.
		 */
		mp->mapptr = cpump;
		cpump->area = kzalloc(PAIE1_CTRBLOCK_SZ, GFP_KERNEL);
		cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL);
		cpump->save = kvmalloc_array(paiext_cnt + 1,
					     sizeof(struct pai_userdata),
					     GFP_KERNEL);
		if (!cpump->save || !cpump->area || !cpump->paiext_cb) {
			paiext_free(mp);
			goto undo;
		}
		refcount_set(&cpump->refcnt, 1);
		cpump->mode = a->sample_period ? PAI_MODE_SAMPLING
					       : PAI_MODE_COUNTING;
	} else {
		/* Multiple invocation, check what is active.
		 * Supported are multiple counter events or only one sampling
		 * event concurrently at any one time.
		 */
		if (cpump->mode == PAI_MODE_SAMPLING ||
		    (cpump->mode == PAI_MODE_COUNTING && a->sample_period)) {
			rc = -EBUSY;
			goto undo;
		}
		refcount_inc(&cpump->refcnt);
	}

	rc = 0;
	cpump->event = event;

undo:
	if (rc) {
		/* Error in allocation of event, decrement anchor. Since
		 * the event in not created, its destroy() function is never
		 * invoked. Adjust the reference counter for the anchor.
		 */
		paiext_root_free();
	}
unlock:
	mutex_unlock(&paiext_reserve_mutex);
	/* If rc is non-zero, no increment of counter/sampler was done. */
	return rc;
}

/* The PAI extension 1 control block supports up to 128 entries. Return
 * the index within PAIE1_CB given the event number. Also validate event
 * number.
 */
static int paiext_event_valid(struct perf_event *event)
{
	u64 cfg = event->attr.config;

	if (cfg >= PAI_NNPA_BASE && cfg <= PAI_NNPA_BASE + paiext_cnt) {
		/* Offset NNPA in paiext_cb */
		event->hw.config_base = offsetof(struct paiext_cb, acc);
		return 0;
	}
	return -EINVAL;
}

/* Might be called on different CPU than the one the event is intended for. */
static int paiext_event_init(struct perf_event *event)
{
	struct perf_event_attr *a = &event->attr;
	int rc;

	/* PMU pai_ext registered as PERF_TYPE_RAW, check event type */
	if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
		return -ENOENT;
	/* PAI extension event must be valid and in supported range */
	rc = paiext_event_valid(event);
	if (rc)
		return rc;
	/* Allow only CPU wide operation, no process context for now. */
	if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)
		return -ENOENT;
	/* Allow only event NNPA_ALL for sampling. */
	if (a->sample_period && a->config != PAI_NNPA_BASE)
		return -EINVAL;
	/* Prohibit exclude_user event selection */
	if (a->exclude_user)
		return -EINVAL;

	rc = paiext_alloc(a, event);
	if (rc)
		return rc;
	event->destroy = paiext_event_destroy;

	if (a->sample_period) {
		a->sample_period = 1;
		a->freq = 0;
		/* Register for paicrypt_sched_task() to be called */
		event->attach_state |= PERF_ATTACH_SCHED_CB;
		/* Add raw data which are the memory mapped counters */
		a->sample_type |= PERF_SAMPLE_RAW;
		/* Turn off inheritance */
		a->inherit = 0;
	}

	return 0;
}

static u64 paiext_getctr(unsigned long *area, int nr)
{
	return area[nr];
}

/* Read the counter values. Return value from location in buffer. For event
 * NNPA_ALL sum up all events.
 */
static u64 paiext_getdata(struct perf_event *event)
{
	struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
	struct paiext_map *cpump = mp->mapptr;
	u64 sum = 0;
	int i;

	if (event->attr.config != PAI_NNPA_BASE)
		return paiext_getctr(cpump->area,
				     event->attr.config - PAI_NNPA_BASE);

	for (i = 1; i <= paiext_cnt; i++)
		sum += paiext_getctr(cpump->area, i);

	return sum;
}

static u64 paiext_getall(struct perf_event *event)
{
	return paie