path: root/fs/resctrl/monitor.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Resource Director Technology(RDT)
 * - Monitoring code
 *
 * Copyright (C) 2017 Intel Corporation
 *
 * Author:
 *    Vikas Shivappa <vikas.shivappa@intel.com>
 *
 * This replaces the cqm.c based on perf but we reuse a lot of
 * code and datastructures originally from Peter Zijlstra and Matt Fleming.
 *
 * More information about RDT be found in the Intel (R) x86 Architecture
 * Software Developer Manual June 2016, volume 3, section 17.17.
 */

#define pr_fmt(fmt)	"resctrl: " fmt

#include <linux/cpu.h>
#include <linux/resctrl.h>
#include <linux/sizes.h>
#include <linux/slab.h>

#include "internal.h"

#define CREATE_TRACE_POINTS

#include "monitor_trace.h"

/**
 * struct rmid_entry - dirty tracking for all RMID.
 * @closid:	The CLOSID for this entry.
 * @rmid:	The RMID for this entry.
 * @busy:	The number of domains with cached data using this RMID.
 * @list:	Member of the rmid_free_lru list when busy == 0.
 *
 * Depending on the architecture the correct monitor is accessed using
 * both @closid and @rmid, or @rmid only.
 *
 * Take the rdtgroup_mutex when accessing.
 */
struct rmid_entry {
	u32				closid;
	u32				rmid;
	int				busy;
	struct list_head		list;
};

/*
 * @rmid_free_lru - A least recently used list of free RMIDs
 *     These RMIDs are guaranteed to have an occupancy less than the
 *     threshold occupancy
 */
static LIST_HEAD(rmid_free_lru);

/*
 * @closid_num_dirty_rmid    The number of dirty RMID each CLOSID has.
 *     Only allocated when CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID is defined.
 *     Indexed by CLOSID. Protected by rdtgroup_mutex.
 */
static u32 *closid_num_dirty_rmid;

/*
 * @rmid_limbo_count - count of currently unused but (potentially)
 *     dirty RMIDs.
 *     This counts RMIDs that no one is currently using but that
 *     may have a occupancy value > resctrl_rmid_realloc_threshold. User can
 *     change the threshold occupancy value.
 */
static unsigned int rmid_limbo_count;

/*
 * @rmid_entry - The entry in the limbo and free lists.
 */
static struct rmid_entry	*rmid_ptrs;

/*
 * This is the threshold cache occupancy in bytes at which we will consider an
 * RMID available for re-allocation.
 */
unsigned int resctrl_rmid_realloc_threshold;

/*
 * This is the maximum value for the reallocation threshold, in bytes.
 */
unsigned int resctrl_rmid_realloc_limit;

/*
 * x86 and arm64 differ in their handling of monitoring.
 * x86's RMID are independent numbers, there is only one source of traffic
 * with an RMID value of '1'.
 * arm64's PMG extends the PARTID/CLOSID space, there are multiple sources of
 * traffic with a PMG value of '1', one for each CLOSID, meaning the RMID
 * value is no longer unique.
 * To account for this, resctrl uses an index. On x86 this is just the RMID,
 * on arm64 it encodes the CLOSID and RMID. This gives a unique number.
 *
 * The domain's rmid_busy_llc and rmid_ptrs[] are sized by index. The arch code
 * must accept an attempt to read every index.
 */
static inline struct rmid_entry *__rmid_entry(u32 idx)
{
	struct rmid_entry *entry;
	u32 closid, rmid;

	entry = &rmid_ptrs[idx];
	resctrl_arch_rmid_idx_decode(idx, &closid, &rmid);

	WARN_ON_ONCE(entry->closid != closid);
	WARN_ON_ONCE(entry->rmid != rmid);

	return entry;
}

static void limbo_release_entry(struct rmid_entry *entry)
{
	lockdep_assert_held(&rdtgroup_mutex);

	rmid_limbo_count--;
	list_add_tail(&entry->list, &rmid_free_lru);

	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID))
		closid_num_dirty_rmid[entry->closid]--;
}

/*
 * Check the RMIDs that are marked as busy for this domain. If the
 * reported LLC occupancy is below the threshold clear the busy bit and
 * decrement the count. If the busy count gets to zero on an RMID, we
 * free the RMID
 */
void __check_limbo(struct rdt_mon_domain *d, bool force_free)
{
	struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
	u32 idx_limit = resctrl_arch_system_num_rmid_idx();
	struct rmid_entry *entry;
	u32 idx, cur_idx = 1;
	void *arch_mon_ctx;
	bool rmid_dirty;
	u64 val = 0;

	arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, QOS_L3_OCCUP_EVENT_ID);
	if (IS_ERR(arch_mon_ctx)) {
		pr_warn_ratelimited("Failed to allocate monitor context: %ld",
				    PTR_ERR(arch_mon_ctx));
		return;
	}

	/*
	 * Skip RMID 0 and start from RMID 1 and check all the RMIDs that
	 * are marked as busy for occupancy < threshold. If the occupancy
	 * is less than the threshold decrement the busy counter of the
	 * RMID and move it to the free list when the counter reaches 0.
	 */
	for (;;) {
		idx = find_next_bit(d->rmid_busy_llc, idx_limit, cur_idx);
		if (idx >= idx_limit)
			break;

		entry = __rmid_entry(idx);
		if (resctrl_arch_rmid_read(r, d, entry->closid, entry->rmid,
					   QOS_L3_OCCUP_EVENT_ID, &val,
					   arch_mon_ctx)) {
			rmid_dirty = true;
		} else {
			rmid_dirty = (val >= resctrl_rmid_realloc_threshold);

			/*
			 * x86's CLOSID and RMID are independent numbers, so the entry's
			 * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the
			 * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't
			 * used to select the configuration. It is thus necessary to track both
			 * CLOSID and RMID because there may be dependencies between them
			 * on some architectures.
			 */
			trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val);
		}

		if (force_free || !rmid_dirty) {
			clear_bit(idx, d->rmid_busy_llc);
			if (!--entry->busy)
				limbo_release_entry(entry);
		}
		cur_idx = idx + 1;
	}

	resctrl_arch_mon_ctx_free(r, QOS_L3_OCCUP_EVENT_ID, arch_mon_ctx);
}

bool has_busy_rmid(struct rdt_mon_domain *d)
{
	u32 idx_limit = resctrl_arch_system_num_rmid_idx();

	return find_first_bit(d->rmid_busy_llc, idx_limit) != idx_limit;
}

static struct rmid_entry *resctrl_find_free_rmid(u32 closid)
{
	struct rmid_entry *itr;
	u32 itr_idx, cmp_idx;

	if (list_empty(&rmid_free_lru))
		return rmid_limbo_count ? ERR_PTR(-EBUSY) : ERR_PTR(-ENOSPC);

	list_for_each_entry(itr, &rmid_free_lru, list) {
		/*
		 * Get the index of this free RMID, and the index it would need
		 * to be if it were used with this CLOSID.
		 * If the CLOSID is irrelevant on this architecture, the two
		 * index values are always the same on every entry and thus the
		 * very first entry will be returned.
		 */
		itr_idx = resctrl_arch_rmid_idx_encode(itr->closid, itr->rmid);
		cmp_idx = resctrl_arch_rmid_idx_encode(closid, itr->rmid);

		if (itr_idx == cmp_idx)
			return itr;
	}

	return ERR_PTR(-ENOSPC);
}

/**
 * resctrl_find_cleanest_closid() - Find a CLOSID where all the associated
 *                                  RMID are clean, or the CLOSID that has
 *                                  the most clean RMID.
 *
 * MPAM's equivalent of RMID are per-CLOSID, meaning a freshly allocated CLOSID
 * may not be able to allocate clean RMID. To avoid this the allocator will
 * choose the CLOSID with the most clean RMID.
 *
 * When the CLOSID and RMID are independent numbers, the first free CLOSID will
 * be returned.
 */
int resctrl_find_cleanest_closid(void)
{
	u32 cleanest_closid = ~0;
	int i = 0;

	lockdep_assert_held(&rdtgroup_mutex);

	if (!IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID))
		return -EIO;

	for (i = 0; i < closids_supported(); i++) {
		int num_dirty;

		if (closid_allocated(i))
			continue;

		num_dirty = closid_num_dirty_rmid[i];
		if (num_dirty == 0)
			return i;

		if (cleanest_closid == ~0)
			cleanest_closid = i;

		if (num_dirty < closid_num_dirty_rmid[cleanest_closid])
			cleanest_closid = i;
	}

	if (cleanest_closid == ~0)
		return -ENOSPC;

	return cleanest_closid;
}

/*
 * For MPAM the RMID value is not unique, and has to be considered with
 * the CLOSID. The (CLOSID, RMID) pair is allocated on all domains, which
 * allows all domains to be managed by a single free list.
 * Each domain also has a rmid_busy_llc to reduce the work of the limbo handler.
 */
int alloc_rmid(u32 closid)
{
	struct rmid_entry *entry;

	lockdep_assert_held(&rdtgroup_mutex);