path: root/arch/x86/events/amd/uncore.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2013 Advanced Micro Devices, Inc.
 *
 * Author: Jacob Shin <jacob.shin@amd.com>
 */

#include <linux/perf_event.h>
#include <linux/percpu.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/cpufeature.h>
#include <linux/smp.h>

#include <asm/perf_event.h>
#include <asm/msr.h>

#define NUM_COUNTERS_NB		4
#define NUM_COUNTERS_L2		4
#define NUM_COUNTERS_L3		6

#define RDPMC_BASE_NB		6
#define RDPMC_BASE_LLC		10

#define COUNTER_SHIFT		16
#define UNCORE_NAME_LEN		16
#define UNCORE_GROUP_MAX	256

#undef pr_fmt
#define pr_fmt(fmt)	"amd_uncore: " fmt

static int pmu_version;

struct amd_uncore_ctx {
	int refcnt;
	int cpu;
	struct perf_event **events;
	struct hlist_node node;
};

struct amd_uncore_pmu {
	char name[UNCORE_NAME_LEN];
	int num_counters;
	int rdpmc_base;
	u32 msr_base;
	int group;
	cpumask_t active_mask;
	struct pmu pmu;
	struct amd_uncore_ctx * __percpu *ctx;
};

enum {
	UNCORE_TYPE_DF,
	UNCORE_TYPE_L3,
	UNCORE_TYPE_UMC,

	UNCORE_TYPE_MAX
};

union amd_uncore_info {
	struct {
		u64	aux_data:32;	/* auxiliary data */
		u64	num_pmcs:8;	/* number of counters */
		u64	gid:8;		/* group id */
		u64	cid:8;		/* context id */
	} split;
	u64		full;
};

struct amd_uncore {
	union amd_uncore_info  __percpu *info;
	struct amd_uncore_pmu *pmus;
	unsigned int num_pmus;
	bool init_done;
	void (*scan)(struct amd_uncore *uncore, unsigned int cpu);
	int  (*init)(struct amd_uncore *uncore, unsigned int cpu);
	void (*move)(struct amd_uncore *uncore, unsigned int cpu);
	void (*free)(struct amd_uncore *uncore, unsigned int cpu);
};

static struct amd_uncore uncores[UNCORE_TYPE_MAX];

static struct amd_uncore_pmu *event_to_amd_uncore_pmu(struct perf_event *event)
{
	return container_of(event->pmu, struct amd_uncore_pmu, pmu);
}

static void amd_uncore_read(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	u64 prev, new;
	s64 delta;

	/*
	 * since we do not enable counter overflow interrupts,
	 * we do not have to worry about prev_count changing on us
	 */

	prev = local64_read(&hwc->prev_count);

	/*
	 * Some uncore PMUs do not have RDPMC assignments. In such cases,
	 * read counts directly from the corresponding PERF_CTR.
	 */
	if (hwc->event_base_rdpmc < 0)
		rdmsrl(hwc->event_base, new);
	else
		rdpmcl(hwc->event_base_rdpmc, new);

	local64_set(&hwc->prev_count, new);
	delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
	delta >>= COUNTER_SHIFT;
	local64_add(delta, &event->count);
}

static void amd_uncore_start(struct perf_event *event, int flags)
{
	struct hw_perf_event *hwc = &event->hw;

	if (flags & PERF_EF_RELOAD)
		wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));

	hwc->state = 0;
	wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
	perf_event_update_userpage(event);
}

static void amd_uncore_stop(struct perf_event *event, int flags)
{
	struct hw_perf_event *hwc = &event->hw;

	wrmsrl(hwc->config_base, hwc->config);
	hwc->state |= PERF_HES_STOPPED;

	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
		event->pmu->read(event);
		hwc->state |= PERF_HES_UPTODATE;
	}
}

static int amd_uncore_add(struct perf_event *event, int flags)
{
	int i;
	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
	struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
	struct hw_perf_event *hwc = &event->hw;

	/* are we already assigned? */
	if (hwc->idx != -1 && ctx->events[hwc->idx] == event)
		goto out;

	for (i = 0; i < pmu->num_counters; i++) {
		if (ctx->events[i] == event) {
			hwc->idx = i;
			goto out;
		}
	}

	/* if not, take the first available counter */
	hwc->idx =