path: root/mm/damon/core.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Data Access Monitor
 *
 * Author: SeongJae Park <sjpark@amazon.de>
 */

#define pr_fmt(fmt) "damon: " fmt

#include <linux/damon.h>
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/string.h>

#define CREATE_TRACE_POINTS
#include <trace/events/damon.h>

#ifdef CONFIG_DAMON_KUNIT_TEST
#undef DAMON_MIN_REGION
#define DAMON_MIN_REGION 1
#endif

static DEFINE_MUTEX(damon_lock);
static int nr_running_ctxs;
static bool running_exclusive_ctxs;

static DEFINE_MUTEX(damon_ops_lock);
static struct damon_operations damon_registered_ops[NR_DAMON_OPS];

static struct kmem_cache *damon_region_cache __ro_after_init;

/* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */
static bool __damon_is_registered_ops(enum damon_ops_id id)
{
	struct damon_operations empty_ops = {};

	if (!memcmp(&empty_ops, &damon_registered_ops[id], sizeof(empty_ops)))
		return false;
	return true;
}

/**
 * damon_is_registered_ops() - Check if a given damon_operations is registered.
 * @id:	Id of the damon_operations to check if registered.
 *
 * Return: true if the ops is set, false otherwise.
 */
bool damon_is_registered_ops(enum damon_ops_id id)
{
	bool registered;

	if (id >= NR_DAMON_OPS)
		return false;
	mutex_lock(&damon_ops_lock);
	registered = __damon_is_registered_ops(id);
	mutex_unlock(&damon_ops_lock);
	return registered;
}

/**
 * damon_register_ops() - Register a monitoring operations set to DAMON.
 * @ops:	monitoring operations set to register.
 *
 * This function registers a monitoring operations set of valid &struct
 * damon_operations->id so that others can find and use them later.
 *
 * Return: 0 on success, negative error code otherwise.
 */
int damon_register_ops(struct damon_operations *ops)
{
	int err = 0;

	if (ops->id >= NR_DAMON_OPS)
		return -EINVAL;
	mutex_lock(&damon_ops_lock);
	/* Fail for already registered ops */
	if (__damon_is_registered_ops(ops->id)) {
		err = -EINVAL;
		goto out;
	}
	damon_registered_ops[ops->id] = *ops;
out:
	mutex_unlock(&damon_ops_lock);
	return err;
}

/**
 * damon_select_ops() - Select a monitoring operations to use with the context.
 * @ctx:	monitoring context to use the operations.
 * @id:		id of the registered monitoring operations to select.
 *
 * This function finds registered monitoring operations set of @id and make
 * @ctx to use it.
 *
 * Return: 0 on success, negative error code otherwise.
 */
int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id)
{
	int err = 0;

	if (id >= NR_DAMON_OPS)
		return -EINVAL;

	mutex_lock(&damon_ops_lock);
	if (!__damon_is_registered_ops(id))
		err = -EINVAL;
	else
		ctx->ops = damon_registered_ops[id];
	mutex_unlock(&damon_ops_lock);
	return err;
}

/*
 * Construct a damon_region struct
 *
 * Returns the pointer to the new struct if success, or NULL otherwise
 */
struct damon_region *damon_new_region(unsigned long start, unsigned long end)
{
	struct damon_region *region;

	region = kmem_cache_alloc(damon_region_cache, GFP_KERNEL);
	if (!region)
		return NULL;

	region->ar.start = start;
	region->ar.end = end;
	region->nr_accesses = 0;
	INIT_LIST_HEAD(&region->list);

	region->age = 0;
	region->last_nr_accesses = 0;

	return region;
}

void damon_add_region(struct damon_region *r, struct damon_target *t)
{
	list_add_tail(&r->list, &t->regions_list);
	t->nr_regions++;
}

static void damon_del_region(struct damon_region *r, struct damon_target *t)
{
	list_del(&r->list);
	t->nr_regions--;
}

static void damon_free_region(struct damon_region *r)
{
	kmem_cache_free(damon_region_cache, r);
}

void damon_destroy_region(struct damon_region *r, struct damon_target *t)
{
	damon_del_region(r, t);
	damon_free_region(r);
}

/*
 * Check whether a region is intersecting an address range
 *
 * Returns true if it is.
 */
static bool damon_intersect(struct damon_region *r,
		struct damon_addr_range *re)
{
	return !(r->ar.end <= re->start || re->end <= r->ar.start);
}

/*
 * Fill holes in regions with new regions.
 */
static int damon_fill_regions_holes(struct damon_region *first,
		struct damon_region *last, struct damon_target *t)
{
	struct damon_region *r = first;

	damon_for_each_region_from(r, t) {
		struct damon_region *next, *newr;

		if (r == last)
			break;
		next = damon_next_region(r);
		if (r->ar.end != next->ar.start) {
			newr = damon_new_region(r->ar.end, next->ar.start);
			if (!newr)
				return -ENOMEM;
			damon_insert_region(newr, r, next, t);
		}
	}
	return 0;
}

/*
 * damon_set_regions() - Set regions of a target for given address ranges.
 * @t:		the given target.
 * @ranges:	array of new monitoring target ranges.
 * @nr_ranges:	length of @ranges.
 *
 * This function adds new regions to, or modify existing regions of a
 * monitoring target to fit in specific ranges.
 *
 * Return: 0 if success, or negative error code otherwise.
 */
int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
		unsigned int nr_ranges)
{
	struct damon_region *r, *next;
	unsigned int i;
	int err;

	/* Remove regions which are not in the new ranges */
	damon_for_each_region_safe(r, next, t) {
		for (i = 0; i < nr_ranges; i++) {
			if (damon_intersect(r, &ranges[i]))
				break;
		}
		if (i == nr_ranges)
			damon_destroy_region(r, t);
	}

	r = damon_first_region(t);
	/* Add new regions or resize existing regions to fit in the ranges */
	for (i = 0; i < nr_ranges; i++) {
		struct damon_region *first = NULL, *last, *newr;
		struct damon_addr_range *range;

		range = &ranges[i];
		/* Get the first/last regions intersecting with the range */
		damon_for_each_region_from(r, t) {
			if (damon_intersect(r, range)) {
				if (!first)
					first = r;
				last = r;
			}
			if (r->ar.start >= range->end)
				break;
		}
		if (!first) {
			/* no region intersects with this range */
			newr = damon_new_region(
					ALIGN_DOWN(range->start,
						DAMON_MIN_REGION),
					ALIGN(range->end, DAMON_MIN_REGION));
			if (!newr)
				return -ENOMEM;
			damon_insert_region(newr, damon_prev_region(r), r, t);
		} else {
			/* resize intersecting regions to fit in this range */
			first->ar.start = ALIGN_DOWN(range->start,
					DAMON_MIN_REGION);
			last->ar.end = ALIGN(range->end, DAMON_MIN_REGION);

			/* fill possible holes in the range */
			err = damon_fill_regions_holes(first, last, t);
			if (err)
				return err;
		}
	}
	return 0;
}

struct damos_filter *damos_new_filter(enum damos_filter_type type,
		bool matching)
{
	struct damos_filter *filter;

	filter = kmalloc(sizeof(*filter), GFP_KERNEL);
	if (!filter)
		return NULL;
	filter->type = type;
	filter->matching = matching;
	INIT_LIST_HEAD(&filter->list);
	return filter;
}

void damos_add_filter(struct damos *s, struct damos_filter *f)
{
	list_add_tail(&f->list, &s->filters);
}

static void damos_del_filter(struct damos_filter *f)
{
	list_del(&f->list);
}

static void damos_free_filter(struct damos_filter *f)
{
	kfree(f);
}

void damos_destroy_filter(struct damos_filter *f)
{
	damos_del_filter(f);
	damos_free_filter(f);
}

/* initialize private fields of damos_quota and return the pointer */
static struct damos_quota *damos_quota_init_priv(struct damos_quota *quota)
{
	quota->total_charged_sz = 0;
	quota->total_charged_ns = 0;
	quota->esz = 0;
	quota->charged_sz = 0;
	quota->ch