path: root/drivers/perf/arm_pmu.c

// SPDX-License-Identifier: GPL-2.0-only
#undef DEBUG

/*
 * ARM performance counter support.
 *
 * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
 * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
 *
 * This code is based on the sparc64 perf event code, which is in turn based
 * on the x86 code.
 */
#define pr_fmt(fmt) "hw perfevents: " fmt

#include <linux/bitmap.h>
#include <linux/cpumask.h>
#include <linux/cpu_pm.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/perf/arm_pmu.h>
#include <linux/slab.h>
#include <linux/sched/clock.h>
#include <linux/spinlock.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>

#include <asm/irq_regs.h>

static int armpmu_count_irq_users(const int irq);

struct pmu_irq_ops {
	void (*enable_pmuirq)(unsigned int irq);
	void (*disable_pmuirq)(unsigned int irq);
	void (*free_pmuirq)(unsigned int irq, int cpu, void __percpu *devid);
};

static void armpmu_free_pmuirq(unsigned int irq, int cpu, void __percpu *devid)
{
	free_irq(irq, per_cpu_ptr(devid, cpu));
}

static const struct pmu_irq_ops pmuirq_ops = {
	.enable_pmuirq = enable_irq,
	.disable_pmuirq = disable_irq_nosync,
	.free_pmuirq = armpmu_free_pmuirq
};

static void armpmu_free_pmunmi(unsigned int irq, int cpu, void __percpu *devid)
{
	free_nmi(irq, per_cpu_ptr(devid, cpu));
}

static const struct pmu_irq_ops pmunmi_ops = {
	.enable_pmuirq = enable_nmi,
	.disable_pmuirq = disable_nmi_nosync,
	.free_pmuirq = armpmu_free_pmunmi
};

static void armpmu_enable_percpu_pmuirq(unsigned int irq)
{
	enable_percpu_irq(irq, IRQ_TYPE_NONE);
}

static void armpmu_free_percpu_pmuirq(unsigned int irq, int cpu,
				   void __percpu *devid)
{
	if (armpmu_count_irq_users(irq) == 1)
		free_percpu_irq(irq, devid);
}

static const struct pmu_irq_ops percpu_pmuirq_ops = {
	.enable_pmuirq = armpmu_enable_percpu_pmuirq,
	.disable_pmuirq = disable_percpu_irq,
	.free_pmuirq = armpmu_free_percpu_pmuirq
};

static void armpmu_enable_percpu_pmunmi(unsigned int irq)
{
	if (!prepare_percpu_nmi(irq))
		enable_percpu_nmi(irq, IRQ_TYPE_NONE);
}

static void armpmu_disable_percpu_pmunmi(unsigned int irq)
{
	disable_percpu_nmi(irq);
	teardown_percpu_nmi(irq);
}

static void armpmu_free_percpu_pmunmi(unsigned int irq, int cpu,
				      void __percpu *devid)
{
	if (armpmu_count_irq_users(irq) == 1)
		free_percpu_nmi(irq, devid);
}

static const struct pmu_irq_ops percpu_pmunmi_ops = {
	.enable_pmuirq = armpmu_enable_percpu_pmunmi,
	.disable_pmuirq = armpmu_disable_percpu_pmunmi,
	.free_pmuirq = armpmu_free_percpu_pmunmi
};

static DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu);
static DEFINE_PER_CPU(int, cpu_irq);
static DEFINE_PER_CPU(const struct pmu_irq_ops *, cpu_irq_ops);

static bool has_nmi;

static inline u64 arm_pmu_event_max_period(struct perf_event *event)
{
	if (event->hw.flags & ARMPMU_EVT_64BIT)
		return GENMASK_ULL(63, 0);
	else
		return GENMASK_ULL(31, 0);
}

static int
armpmu_map_cache_event(const unsigned (*cache_map)
				      [PERF_COUNT_HW_CACHE_MAX]
				      [PERF_COUNT_HW_CACHE_OP_MAX]
				      [PERF_COUNT_HW_CACHE_RESULT_MAX],
		       u64 config)
{
	unsigned int cache_type, cache_op, cache_result, ret;

	cache_type = (config >>  0) & 0xff;
	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
		return -EINVAL;

	cache_op = (config >>  8) & 0xff;
	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
		return -EINVAL;

	cache_result = (config >> 16) & 0xff;
	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
		return -EINVAL;

	if (!cache_map)
		return -ENOENT;

	ret = (int)(*cache_map)[cache_type][cache_op][cache_result];

	if (ret == CACHE_OP_UNSUPPORTED)
		return -ENOENT;

	return ret;
}

static int
armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
{
	int mapping;

	if (config >= PERF_COUNT_HW_MAX)
		return -EINVAL;

	if (!event_map)
		return -ENOENT;

	mapping = (*event_map)[config];
	return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
}

static int
armpmu_map_raw_event(u32 raw_event_mask, u64 config)
{
	return (int)(config & raw_event_mask);
}

int
armpmu_map_event(struct perf_event *event,
		 const unsigned (*event_map)[PERF_COUNT_HW_MAX],
		 const unsigned (*cache_map)
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX],
		 u32 raw_event_mask)
{
	u64 config = event->attr.config;
	int type = event->attr.type;

	if (type == event->pmu->type)
		return armpmu_map_raw_event(raw_event_mask, config);

	switch (type) {
	case PERF_TYPE_HARDWARE:
		return armpmu_map_hw_event(event_map, config);
	case PERF_TYPE_HW_CACHE:
		return armpmu_map_cache_event(cache_map, config);
	case PERF_TYPE_RAW:
		return armpmu_map_raw_event(raw_event_mask, config);
	}

	return -ENOENT;
}

int a