path: root/kernel/rcu/refscale.c

// SPDX-License-Identifier: GPL-2.0+
//
// Scalability test comparing RCU vs other mechanisms
// for acquiring references on objects.
//
// Copyright (C) Google, 2020.
//
// Author: Joel Fernandes <joel@joelfernandes.org>

#define pr_fmt(fmt) fmt

#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/completion.h>
#include <linux/cpu.h>
#include <linux/delay.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kthread.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/rcupdate_trace.h>
#include <linux/reboot.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/stat.h>
#include <linux/srcu.h>
#include <linux/slab.h>
#include <linux/torture.h>
#include <linux/types.h>

#include "rcu.h"

#define SCALE_FLAG "-ref-scale: "

#define SCALEOUT(s, x...) \
	pr_alert("%s" SCALE_FLAG s, scale_type, ## x)

#define VERBOSE_SCALEOUT(s, x...) \
	do { \
		if (verbose) \
			pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
	} while (0)

static atomic_t verbose_batch_ctr;

#define VERBOSE_SCALEOUT_BATCH(s, x...)							\
do {											\
	if (verbose &&									\
	    (verbose_batched <= 0 ||							\
	     !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) {		\
		schedule_timeout_uninterruptible(1);					\
		pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x);			\
	}										\
} while (0)

#define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");

static char *scale_type = "rcu";
module_param(scale_type, charp, 0444);
MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");

torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");

// Wait until there are multiple CPUs before starting test.
torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
	      "Holdoff time before test start (s)");
// Number of typesafe_lookup structures, that is, the degree of concurrency.
torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");
// Number of loops per experiment, all readers execute operations concurrently.
torture_param(long, loops, 10000, "Number of loops per experiment.");
// Number of readers, with -1 defaulting to about 75% of the CPUs.
torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
// Number of runs.
torture_param(int, nruns, 30, "Number of experiments to run.");
// Reader delay in nanoseconds, 0 for no delay.
torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");

#ifdef MODULE
# define REFSCALE_SHUTDOWN 0
#else
# define REFSCALE_SHUTDOWN 1
#endif

torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
	      "Shutdown at end of scalability tests.");

struct reader_task {
	struct task_struct *task;
	int start_reader;
	wait_queue_head_t wq;
	u64 last_duration_ns;
};

static struct task_struct *shutdown_task;
static wait_queue_head_t shutdown_wq;

static struct task_struct *main_task;
static wait_queue_head_t main_wq;
static int shutdown_start;

static struct reader_task *reader_tasks;

// Number of readers that are part of the current experiment.
static atomic_t nreaders_exp;

// Use to wait for all threads to start.
static atomic_t n_init;
static atomic_t n_started;
static atomic_t n_warmedup;
static atomic_t n_cooleddown;

// Track which experiment is currently running.
static int exp_idx;

// Operations vector for selecting different types of tests.
struct ref_scale_ops {
	bool (*init)(void);
	void (*cleanup)(void);
	void (*readsection)(const int nloops);
	void (*delaysection)(const int nloops, const int udl, const int ndl);
	const char *name;
};

static struct ref_scale_ops *cur_ops;

static void un_delay(const int udl, const int ndl)
{
	if (udl)
		udelay(udl);
	if (ndl)
		ndelay(ndl);
}

static void ref_rcu_read_section(const int nloops)
{
	int i;

	for (i = nloops; i >= 0; i--) {
		rcu_read_lock();
		rcu_read_unlock();
	}
}

static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
{
	int i;

	for (i = nloops; i >= 0; i--) {
		rcu_read_lock();
		un_delay(udl, ndl);
		rcu_read_unlock();
	}
}

static bool rcu_sync_scale_init(void)
{
	return true