path: root/kernel/rcu/rcuscale.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * Read-Copy Update module-based scalability-test facility
 *
 * Copyright (C) IBM Corporation, 2015
 *
 * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
 */

#define pr_fmt(fmt) fmt

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/err.h>
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/rcupdate.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <uapi/linux/sched/types.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/completion.h>
#include <linux/moduleparam.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
#include <linux/reboot.h>
#include <linux/freezer.h>
#include <linux/cpu.h>
#include <linux/delay.h>
#include <linux/stat.h>
#include <linux/srcu.h>
#include <linux/slab.h>
#include <asm/byteorder.h>
#include <linux/torture.h>
#include <linux/vmalloc.h>
#include <linux/rcupdate_trace.h>
#include <linux/sched/debug.h>

#include "rcu.h"

MODULE_DESCRIPTION("Read-Copy Update module-based scalability-test facility");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");

#define SCALE_FLAG "-scale:"
#define SCALEOUT_STRING(s) \
	pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s)
#define VERBOSE_SCALEOUT_STRING(s) \
	do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0)
#define SCALEOUT_ERRSTRING(s) \
	pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s)

/*
 * The intended use cases for the nreaders and nwriters module parameters
 * are as follows:
 *
 * 1.	Specify only the nr_cpus kernel boot parameter.  This will
 *	set both nreaders and nwriters to the value specified by
 *	nr_cpus for a mixed reader/writer test.
 *
 * 2.	Specify the nr_cpus kernel boot parameter, but set
 *	rcuscale.nreaders to zero.  This will set nwriters to the
 *	value specified by nr_cpus for an update-only test.
 *
 * 3.	Specify the nr_cpus kernel boot parameter, but set
 *	rcuscale.nwriters to zero.  This will set nreaders to the
 *	value specified by nr_cpus for a read-only test.
 *
 * Various other use cases may of course be specified.
 *
 * Note that this test's readers are intended only as a test load for
 * the writers.  The reader scalability statistics will be overly
 * pessimistic due to the per-critical-section interrupt disabling,
 * test-end checks, and the pair of calls through pointers.
 */

#ifdef MODULE
# define RCUSCALE_SHUTDOWN 0
#else
# define RCUSCALE_SHUTDOWN 1
#endif

torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives");
torture_param(int, gp_async_max, 1000, "Max # outstanding waits per writer");
torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
torture_param(int, holdoff, 10, "Holdoff time before test start (s)");
torture_param(int, minruntime, 0, "Minimum run time (s)");
torture_param(int, nreaders, -1, "Number of RCU reader threads");
torture_param(int, nwriters, -1, "Number of RCU updater threads");
torture_param(bool, shutdown, RCUSCALE_SHUTDOWN,
	      "Shutdown at end of scalability tests.");
torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable");
torture_param(int, writer_holdoff_jiffies, 0, "Holdoff (jiffies) between GPs, zero to disable");
torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?");
torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate.");
torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?");

static char *scale_type = "rcu";
module_param(scale_type, charp, 0444);
MODULE_PARM_DESC(scale_type, "Type of RCU to scalability-test (rcu, srcu, ...)");

// Structure definitions for custom fixed-per-task allocator.
struct writer_mblock {
	struct rcu_head wmb_rh;
	struct llist_node wmb_node;
	struct writer_freelist *wmb_wfl;
};

struct writer_freelist {
	struct llist_head ws_lhg;
	atomic_t ws_inflight;
	struct llist_head ____cacheline_internodealigned_in_smp ws_lhp;
	struct writer_mblock *ws_mblocks;
};

static int nrealreaders;
static int nrealwriters;
static struct task_struct **writer_tasks;
static struct task_struct **reader_tasks;
static struct task_struct *shutdown_task;

static u64 **writer_durations;
static bool *writer_done;
static struct writer_freelist *writer_freelists;
static int *writer_n_durations;
static atomic_t n_rcu_scale_reader_started;
static atomic_t n_rcu_scale_writer_started;
static atomic_t n_rcu_scale_writer_finished;
static wait_queue_head_t shutdown_wq;
static u64 t_rcu_scale_writer_started;
static u64