path: root/tools/testing/selftests/mm/uffd-stress.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Stress userfaultfd syscall.
 *
 *  Copyright (C) 2015  Red Hat, Inc.
 *
 * This test allocates two virtual areas and bounces the physical
 * memory across the two virtual areas (from area_src to area_dst)
 * using userfaultfd.
 *
 * There are three threads running per CPU:
 *
 * 1) one per-CPU thread takes a per-page pthread_mutex in a random
 *    page of the area_dst (while the physical page may still be in
 *    area_src), and increments a per-page counter in the same page,
 *    and checks its value against a verification region.
 *
 * 2) another per-CPU thread handles the userfaults generated by
 *    thread 1 above. userfaultfd blocking reads or poll() modes are
 *    exercised interleaved.
 *
 * 3) one last per-CPU thread transfers the memory in the background
 *    at maximum bandwidth (if not already transferred by thread
 *    2). Each cpu thread takes cares of transferring a portion of the
 *    area.
 *
 * When all threads of type 3 completed the transfer, one bounce is
 * complete. area_src and area_dst are then swapped. All threads are
 * respawned and so the bounce is immediately restarted in the
 * opposite direction.
 *
 * per-CPU threads 1 by triggering userfaults inside
 * pthread_mutex_lock will also verify the atomicity of the memory
 * transfer (UFFDIO_COPY).
 */

#include "uffd-common.h"

#ifdef __NR_userfaultfd

#define BOUNCE_RANDOM		(1<<0)
#define BOUNCE_RACINGFAULTS	(1<<1)
#define BOUNCE_VERIFY		(1<<2)
#define BOUNCE_POLL		(1<<3)
static int bounces;

/* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
#define ALARM_INTERVAL_SECS 10
static char *zeropage;
pthread_attr_t attr;

#define swap(a, b) \
	do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)

#define factor_of_2(x) ((x) ^ ((x) & ((x) - 1)))

const char *examples =
    "# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
    "./userfaultfd anon 100 99999\n\n"
    "# Run the same anonymous memory test, but using /dev/userfaultfd:\n"
    "./userfaultfd anon:dev 100 99999\n\n"
    "# Run share memory test on 1GiB region with 99 bounces:\n"
    "./userfaultfd shmem 1000 99\n\n"
    "# Run hugetlb memory test on 256MiB region with 50 bounces:\n"
    "./userfaultfd hugetlb 256 50\n\n"
    "# Run the same hugetlb test but using shared file:\n"
    "./userfaultfd hugetlb_shared 256 50\n\n"
    "# 10MiB-~6GiB 999 bounces anonymous test, "
    "continue forever unless an error triggers\n"
    "while ./userfaultfd anon $[RANDOM % 6000 + 10] 999; do true; done\n\n";

static void usage(void)
{
	fprintf(stderr, "\nUsage: ./userfaultfd <test type> <MiB> <bounces> "
		"[hugetlbfs_file]\n\n");
	fprintf(stderr, "Supported <test type>: anon, hugetlb, "
		"hugetlb_shared, shmem\n\n");
	fprintf(stderr, "'Test mods' can be joined to the test type string with a ':'. "
		"Supported mods:\n");
	fprintf(stderr, "\tsyscall - Use userfaultfd(2) (default)\n");
	fprintf(stderr, "\tdev - Use /dev/userfaultfd instead of userfaultfd(2)\n");
	fprintf(stderr, "\tcollapse - Test MADV_COLLAPSE of UFFDIO_REGISTER_MODE_MINOR\n"
		"memory\n");
	fprintf(stderr, "\nExample test mod usage:\n");
	fprintf(stderr, "# Run anonymous memory test with /dev/userfaultfd:\n");
	fprintf(stderr, "./userfaultfd anon:dev 100 99999\n\n");

	fprintf(stderr, "Examples:\n\n");
	fprintf(stderr, "%s", examples);
	exit(1);
}

static void uffd_stats_reset(struct uffd_args *args, unsigned long n_cpus)
{
	int i;

	for (i = 0; i < n_cpus; i++) {
		args[i].cpu = i;
		args[i].apply_wp = test_uffdio_wp;
		args[i].missing_faults = 0;
		args[i].wp_faults = 0;
		args[i].minor_faults = 0;
	}
}

static inline uint64_t uffd_minor_feature(void)
{
	if (test_type == TEST_HUGETLB && map_shared)
		return UFFD_FEATURE_MINOR_HUGETLBFS;
	else if (test_type == TEST_SHMEM)
		return UFFD_FEATURE_MINOR_SHMEM;
	else
		return 0;
}

static int my_bcmp(char *str1, char *str2, size_t n)
{
	unsigned long i;
	for (i = 0; i < n; i++)
		if (str1[i] != str2[i])
			return 1;
	return 0;
}

static void *locking_thread(void *arg)
{
	unsigned long cpu = (unsigned long) arg;
	unsigned long page_nr;
	unsigned long long count;

	if (!(bounces & BOUNCE_RANDOM)) {
		page_nr = -bounces;
		if (!(bounces & BOUNCE_RACINGFAULTS))
			page_nr += cpu * nr_pages_per_cpu;
	}

	while (!finished) {
		if (bounces & BOUNCE_RANDOM) {
			if (getrandom(&page_nr, sizeof(page_nr), 0) != sizeof(page_nr))
				err("getrandom failed");
		} else
			page_nr += 1;
		page_nr %= nr_pages;
		pthread_mutex_lock(area_mutex(area_dst, page_nr));
		count = *area_count(area_dst, page_nr);
		if (count != count_verify[page_nr])
			err("page_nr %lu memory corruption %llu %llu",
			    page_nr, count, count_verify[page_nr]);
		count++;
		*area_count(area_dst, page_nr) = count_verify[page_nr] = count;
		pthread_mutex_unlock(area_mutex(area_dst, page_nr));
	}

	return NULL;
}

static int co