path: root/mm/damon/core.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Data Access Monitor
 *
 * Author: SeongJae Park <sjpark@amazon.de>
 */

#define pr_fmt(fmt) "damon: " fmt

#include <linux/damon.h>
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/string.h>

#define CREATE_TRACE_POINTS
#include <trace/events/damon.h>

#ifdef CONFIG_DAMON_KUNIT_TEST
#undef DAMON_MIN_REGION
#define DAMON_MIN_REGION 1
#endif

static DEFINE_MUTEX(damon_lock);
static int nr_running_ctxs;
static bool running_exclusive_ctxs;

static DEFINE_MUTEX(damon_ops_lock);
static struct damon_operations damon_registered_ops[NR_DAMON_OPS];

static struct kmem_cache *damon_region_cache __ro_after_init;

/* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */
static bool __damon_is_registered_ops(enum damon_ops_id id)
{
	struct damon_operations empty_ops = {};

	if (!memcmp(&empty_ops, &damon_registered_ops[id], sizeof(empty_ops)))
		return false;
	return true;
}

/**
 * damon_is_registered_ops() - Check if a given damon_operations is registered.
 * @id:	Id of the damon_operations to check if registered.
 *
 * Return: true if the ops is set, false otherwise.
 */
bool damon_is_registered_ops(enum damon_ops_id id)
{
	bool registered;

	if (id >= NR_DAMON_OPS)
		return false;
	mutex_lock(&damon_ops_lock);
	registered = __damon_is_registered_ops(id);
	mutex_unlock(&damon_ops_lock);
	return registered;
}

/**
 * damon_register_ops() - Register a monitoring operations set to DAMON.
 * @ops:	monitoring operations set to register.
 *
 * This function registers a monitoring operations set of valid &struct
 * damon_operations->id so that others can find and use them later.
 *
 * Return: 0 on success, negative error code otherwise.
 */
int damon_register_ops(struct damon_operations *ops)
{
	int err = 0;

	if (ops->id >= NR_DAMON_OPS)
		return -EINVAL;
	mutex_lock(&damon_ops_lock);
	/* Fail for already registered ops */
	if (__damon_is_registered_ops(ops->id)) {
		err = -EINVAL;
		goto out;
	}
	damon_registered_ops[ops->id] = *ops;
out:
	mutex_unlock(&damon_ops_lock);
	return err;
}

/**
 * damon_select_ops() - Select a monitoring operations to use with the context.
 * @ctx:	monitoring context to use the operations.
 * @id:		id of the registered monitoring operations to select.
 *
 * This function finds registered monitoring operations set of @id and make
 * @ctx to use it.
 *
 * Return: 0 on success, negative error code otherwise.
 */
int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id)
{
	int err = 0;

	if (id >= NR_DAMON_OPS)
		return -EINVAL;

	mutex_lock(&damon_ops_lock);
	if (!__damon_is_registered_ops(id))
		err = -EINVAL;
	else
		ctx->ops = damon_registered_ops[id];
	mutex_unlock(&damon_ops_lock);
	return err;
}

/*
 * Construct a damon_region struct
 *
 * Returns the pointer to the new struct if success, or NULL otherwise
 */
struct damon_region *damon_new_region(unsigned long start, unsigned long end)
{
	struct damon_region *region;

	region = kmem_cache_alloc(damon_region_cache, GFP_KERNEL);
	if (!region)
		return NULL;

	region->ar.start = start;
	region->ar.end = end;
	region->nr_accesses = 0;
	INIT_LIST_HEAD(&region->list);

	region->age = 0;
	region->last_nr_accesses = 0;

	return region;
}

void damon_add_region(struct damon_region *r, struct damon_target *t)
{
	list_add_tail(&r->list, &t->regions_list);
	t->nr_regions++;
}

static void damon_del_region(struct damon_region *r, struct damon_target *t)
{
	list_del(&r->list);
	t->nr_regions--;
}

static void damon_free_region(struct damon_region *r)
{
	kmem_cache_free(damon_region_cache, r);
}

void damon_destroy_region(struct damon_region *r, struct damon_target *t)
{
	damon_del_region(r, t);
	damon_free_region(r);
}

/*
 * Check whether a region is intersecting an address range
 *
 * Returns true if it is.
 */
static bool damon_intersect(struct damon_region *r,
		struct damon_addr_range *re)
{
	return !(r->ar.end <= re->start || re->end <= r->ar.start);
}

/*
 * Fill holes in regions with new regions.
 */
static int damon_fill_regions_holes(struct damon_region *first,
		struct damon_region *last, struct damon_target *t)
{
	struct damon_region *r = first;

	damon_for_each_region_from(r, t) {
		struct damon_region *next, *newr;

		if (r == last)
			break;
		next = damon_next_region(r);
		if (r->ar.end != next->ar.start) {
			newr = damon_new_region(r->ar.end, next->ar.start);
			if (!newr)
				return -ENOMEM;
			damon_insert_region(newr, r, next, t);
		}
	}
	return 0;
}

/*
 * damon_set_regions() - Set regions of a target for given address ranges.
 * @t:		the given target.
 * @ranges:	array of new monitoring target ranges.
 * @nr_ranges:	length of @ranges.
 *
 * This function adds new regions to, or modify existing regions of a
 * monitoring target to fit in specific ranges.
 *
 * Return: 0 if success, or negative error code ot