path: root/tools/testing/selftests/vm/userfaultfd.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Stress userfaultfd syscall.
 *
 *  Copyright (C) 2015  Red Hat, Inc.
 *
 * This test allocates two virtual areas and bounces the physical
 * memory across the two virtual areas (from area_src to area_dst)
 * using userfaultfd.
 *
 * There are three threads running per CPU:
 *
 * 1) one per-CPU thread takes a per-page pthread_mutex in a random
 *    page of the area_dst (while the physical page may still be in
 *    area_src), and increments a per-page counter in the same page,
 *    and checks its value against a verification region.
 *
 * 2) another per-CPU thread handles the userfaults generated by
 *    thread 1 above. userfaultfd blocking reads or poll() modes are
 *    exercised interleaved.
 *
 * 3) one last per-CPU thread transfers the memory in the background
 *    at maximum bandwidth (if not already transferred by thread
 *    2). Each cpu thread takes cares of transferring a portion of the
 *    area.
 *
 * When all threads of type 3 completed the transfer, one bounce is
 * complete. area_src and area_dst are then swapped. All threads are
 * respawned and so the bounce is immediately restarted in the
 * opposite direction.
 *
 * per-CPU threads 1 by triggering userfaults inside
 * pthread_mutex_lock will also verify the atomicity of the memory
 * transfer (UFFDIO_COPY).
 */

#define _GNU_SOURCE
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <time.h>
#include <signal.h>
#include <poll.h>
#include <string.h>
#include <linux/mman.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <sys/ioctl.h>
#include <sys/wait.h>
#include <pthread.h>
#include <linux/userfaultfd.h>
#include <setjmp.h>
#include <stdbool.h>
#include <assert.h>

#include "../kselftest.h"

#ifdef __NR_userfaultfd

static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;

#define BOUNCE_RANDOM		(1<<0)
#define BOUNCE_RACINGFAULTS	(1<<1)
#define BOUNCE_VERIFY		(1<<2)
#define BOUNCE_POLL		(1<<3)
static int bounces;

#define TEST_ANON	1
#define TEST_HUGETLB	2
#define TEST_SHMEM	3
static int test_type;

/* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
#define ALARM_INTERVAL_SECS 10
static volatile bool test_uffdio_copy_eexist = true;
static volatile bool test_uffdio_zeropage_eexist = true;
/* Whether to test uffd write-protection */
static bool test_uffdio_wp = false;

static bool map_shared;
static int huge_fd;
static char *huge_fd_off0;
static unsigned long long *count_verify;
static int uffd, uffd_flags, finished, *pipefd;
static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;
static char *zeropage;
pthread_attr_t attr;

/* Userfaultfd test statistics */
struct uffd_stats {
	int cpu;
	unsigned long missing_faults;
	unsigned long wp_faults;
};

/* pthread_mutex_t starts at page offset 0 */
#define area_mutex(___area, ___nr)					\
	((pthread_mutex_t *) ((___area) + (___nr)*page_size))
/*
 * count is placed in the page after pthread_mutex_t naturally aligned
 * to avoid non alignment faults on non-x86 archs.
 */
#define area_count(___area, ___nr)					\
	((volatile unsigned long long *) ((unsigned long)		\
				 ((___area) + (___nr)*page_size +	\
				  sizeof(pthread_mutex_t) +		\
				  sizeof(unsigned long long) - 1) &	\
				 ~(unsigned long)(sizeof(unsigned long long) \
						  -  1)))

const char *examples =
    "# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
    "./userfaultfd anon 100 99999\n\n"
    "# Run share memory test on 1GiB region with 99 bounces:\n"
    "./userfaultfd shmem 1000 99\n\n"
    "# Run hugetlb memory test on 256MiB region with 50 bounces (using /dev/hugepages/hugefile):\n"
    "./userfaultfd hugetlb 256 50 /dev/hugepages/hugefile\n\n"
    "# Run the same hugetlb test but using shmem:\n"
    "./userfaultfd hugetlb_shared 256 50 /dev/hugepages/hugefile\n\n"
    "# 10MiB-~6GiB 999 bounces anonymous test, "
    "continue forever unless an error triggers\n"
    "while ./userfaultfd anon $[RANDOM % 6000 + 10] 999; do true; done\n\n";

static void usage(void)
{
	fprintf(stderr, "\nUsage: ./userfaultfd <test type> <MiB> <bounces> "
		"[hugetlbfs_file]\n\n");
	fprintf(stderr, "Supported <test type>: anon, hugetlb, "
		"hugetlb_shared, shmem\n\n");
	fprintf(stderr, "Examples:\n\n");
	fprintf(stderr, "%s", examples);
	exit(1);
}

static void uffd_stats_reset(struct uffd_stats *uffd_stats,
			     unsigned long n_cpus)
{
	int i;

	for (i = 0; i < n_cpus; i++) {
		uffd_stats[i].cpu = i;
		uffd_stats[i].missing_faults = 0;
		uffd_stats[i].wp_faults = 0;
	}
}

static void uffd_stats_report(struct uffd_stats *stats, int n_cpus)
{
	int i;
	unsigned long long miss_total = 0, wp_total = 0;

	for (i = 0; i < n_cpus; i++) {
		miss_total += stats[i].missing_faults;
		wp_total += stats[i].wp_faults;
	}

	printf("userfaults: %llu missing (", miss_total);
	for (i = 0; i < n_cpus; i++)
		printf("%lu+", stats[i].missing_faults);
	printf("\b), %llu wp (", wp_total);
	for (i = 0; i < n_cpus; i++)
		printf("%lu+", stats[i].wp_faults);
	printf("\b)\n");
}

static int anon_release_pages(char *rel_area)
{
	int ret = 0;

	if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) {
		perror("madvise");
		ret = 1;
	}

	return ret;
}

static void anon_allocate_area(void **alloc_area)
{
	*alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
	if (*alloc_area == MAP_FAILED) {
		fprintf(stderr, "mmap of anonymous memory failed");
		*alloc_area = NULL;
	}
}

static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
{
}

/* HugeTLB memory */
static int hugetlb_release_pages(char *rel_area)
{
	int ret = 0;

	if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
				rel_area == huge_fd_off0 ? 0 :
				nr_pages * page_size,
				nr_pages * page_size)) {
		perror("fallocate");
		ret = 1;
	}

	return ret;
}

static void hugetlb_allocat