xref: /linux/tools/testing/selftests/mm/uffd-unit-tests.c (revision 1d95be59ca0698c1095a08e11b6bc60df7bfaa26)

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Userfaultfd unit tests.
 *
 *  Copyright (C) 2015-2023  Red Hat, Inc.
 */

#include "uffd-common.h"

#include "../../../../mm/gup_test.h"

#ifdef __NR_userfaultfd

/* The unit test doesn't need a large or random size, make it 32MB for now */
#define  UFFD_TEST_MEM_SIZE               (32UL << 20)

#define  MEM_ANON                         BIT_ULL(0)
#define  MEM_SHMEM                        BIT_ULL(1)
#define  MEM_SHMEM_PRIVATE                BIT_ULL(2)
#define  MEM_HUGETLB                      BIT_ULL(3)
#define  MEM_HUGETLB_PRIVATE              BIT_ULL(4)

#define  MEM_ALL  (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
		   MEM_HUGETLB | MEM_HUGETLB_PRIVATE)

#define ALIGN_UP(x, align_to) \
	((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))

#define MAX(a, b) (((a) > (b)) ? (a) : (b))

struct mem_type {
	const char *name;
	unsigned int mem_flag;
	uffd_test_ops_t *mem_ops;
	bool shared;
};
typedef struct mem_type mem_type_t;

mem_type_t mem_types[] = {
	{
		.name = "anon",
		.mem_flag = MEM_ANON,
		.mem_ops = &anon_uffd_test_ops,
		.shared = false,
	},
	{
		.name = "shmem",
		.mem_flag = MEM_SHMEM,
		.mem_ops = &shmem_uffd_test_ops,
		.shared = true,
	},
	{
		.name = "shmem-private",
		.mem_flag = MEM_SHMEM_PRIVATE,
		.mem_ops = &shmem_uffd_test_ops,
		.shared = false,
	},
	{
		.name = "hugetlb",
		.mem_flag = MEM_HUGETLB,
		.mem_ops = &hugetlb_uffd_test_ops,
		.shared = true,
	},
	{
		.name = "hugetlb-private",
		.mem_flag = MEM_HUGETLB_PRIVATE,
		.mem_ops = &hugetlb_uffd_test_ops,
		.shared = false,
	},
};

/* Arguments to be passed over to each uffd unit test */
struct uffd_test_args {
	mem_type_t *mem_type;
};
typedef struct uffd_test_args uffd_test_args_t;

/* Returns: UFFD_TEST_* */
typedef void (*uffd_test_fn)(uffd_global_test_opts_t *, uffd_test_args_t *);

typedef struct {
	const char *name;
	uffd_test_fn uffd_fn;
	unsigned int mem_targets;
	uint64_t uffd_feature_required;
	uffd_test_case_ops_t *test_case_ops;
} uffd_test_case_t;

static char current_test[256];

static void uffd_test_pass(void)
{
	ksft_test_result_pass("%s\n", current_test);
}

#define  uffd_test_start(...)  do {		\
		snprintf(current_test, sizeof(current_test), __VA_ARGS__); \
	} while (0)

#define  uffd_test_fail(fmt, ...)  do {					\
		ksft_print_msg("failed reason: [" fmt "]\n", ##__VA_ARGS__); \
		ksft_test_result_fail("%s\n", current_test);		\
	} while (0)

static void uffd_test_skip(const char *message)
{
	ksft_test_result_skip("%s (%s)\n", current_test, message);
}

static void test_uffd_api(bool use_dev)
{
	struct uffdio_api uffdio_api;
	int uffd;

	uffd_test_start("UFFDIO_API (with %s)",
			use_dev ? "/dev/userfaultfd" : "syscall");

	if (use_dev)
		uffd = uffd_open_dev(UFFD_FLAGS);
	else
		uffd = uffd_open_sys(UFFD_FLAGS);
	if (uffd < 0) {
		uffd_test_skip("cannot open userfaultfd handle");
		return;
	}

	/* Test wrong UFFD_API */
	uffdio_api.api = 0xab;
	uffdio_api.features = 0;
	if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
		uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
		goto out;
	}

	/* Test wrong feature bit */
	uffdio_api.api = UFFD_API;
	uffdio_api.features = BIT_ULL(63);
	if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
		uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
		goto out;
	}

	/* Test normal UFFDIO_API */
	uffdio_api.api = UFFD_API;
	uffdio_api.features = 0;
	if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
		uffd_test_fail("UFFDIO_API should succeed but failed");
		goto out;
	}

	/* Test double requests of UFFDIO_API with a random feature set */
	uffdio_api.features = BIT_ULL(0);
	if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
		uffd_test_fail("UFFDIO_API should reject initialized uffd");
		goto out;
	}

	uffd_test_pass();
out:
	close(uffd);
}


static bool uffd_feature_supported(uffd_test_case_t *test)
{
	uint64_t features;

	if (uffd_get_features(&features))
		return false;

	return (features & test->uffd_feature_required) ==
	    test->uffd_feature_required;
}

static int pagemap_open(void)
{
	int fd = open("/proc/self/pagemap", O_RDONLY);

	if (fd < 0)
		err("open pagemap");

	return fd;
}

/* This macro let __LINE__ works in err() */
#define  pagemap_check_wp(value, wp) do {				\
		if (!!(value & PM_UFFD_WP) != wp)			\
			err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
	} while (0)

typedef struct {
	uffd_global_test_opts_t *gopts;
	int child_uffd;
} fork_event_args;

static void *fork_event_consumer(void *data)
{
	fork_event_args *args = data;
	struct uffd_msg msg = { 0 };

	args->gopts->ready_for_fork = true;

	/* Read until a full msg received */
	while (uffd_read_msg(args->gopts, &msg));

	if (msg.event != UFFD_EVENT_FORK)
		err("wrong message: %u\n", msg.event);

	/* Just to be properly freed later */
	args->child_uffd = msg.arg.fork.ufd;
	return NULL;
}

typedef struct {
	int gup_fd;
	bool pinned;
} pin_args;

/*
 * Returns 0 if succeed, <0 for errors.  pin_pages() needs to be paired
 * with unpin_pages().  Currently it needs to be RO longterm pin to satisfy
 * all needs of the test cases (e.g., trigger unshare, trigger fork() early
 * CoW, etc.).
 */
static int pin_pages(pin_args *args, void *buffer, size_t size)
{
	struct pin_longterm_test test = {
		.addr = (uintptr_t)buffer,
		.size = size,
		/* Read-only pins */
		.flags = 0,
	};

	if (args->pinned)
		err("already pinned");

	args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
	if (args->gup_fd < 0)
		return -errno;

	if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
		/* Even if gup_test existed, can be an old gup_test / kernel */
		close(args->gup_fd);
		return -errno;
	}
	args->pinned = true;
	return 0;
}

static void unpin_pages(pin_args *args)
{
	if (!args->pinned)
		err("unpin without pin first");
	if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
		err("PIN_LONGTERM_TEST_STOP");
	close(args->gup_fd);
	args->pinned = false;
}

static int pagemap_test_fork(uffd_global_test_opts_t *gopts, bool with_event, bool test_pin)
{
	fork_event_args args = { .gopts = gopts, .child_uffd = -1 };
	pthread_t thread;
	pid_t child;
	uint64_t value;
	int fd, result;

	/* Prepare a thread to resolve EVENT_FORK */
	if (with_event) {
		gopts->ready_for_fork = false;
		if (pthread_create(&thread, NULL, fork_event_consumer, &args))
			err("pthread_create()");
		while (!gopts->ready_for_fork)
			; /* Wait for the poll_thread to start executing before forking */
	}

	child = fork();
	if (!child) {
		/* Open the pagemap fd of the child itself */
		pin_args args = {};

		fd = pagemap_open();

		if (test_pin && pin_pages(&args, gopts->area_dst, gopts->page_size))
			/*
			 * Normally when reach here we have pinned in
			 * previous tests, so shouldn't fail anymore
			 */
			err("pin page failed in child");

		value = pagemap_get_entry(fd, gopts->area_dst);
		/*
		 * After fork(), we should handle uffd-wp bit differently:
		 *
		 * (1) when with EVENT_FORK, it should persist
		 * (2) when without EVENT_FORK, it should be dropped
		 */
		pagemap_check_wp(value, with_event);
		if (test_pin)
			unpin_pages(&args);
		/* Succeed */
		_exit(0);
	}
	waitpid(child, &result, 0);

	if (with_event) {
		if (pthread_join(thread, NULL))
			err("pthread_join()");
		if (args.child_uffd < 0)
			err("Didn't receive child uffd");
		close(args.child_uffd);
	}

	return result;
}

static void uffd_wp_unpopulated_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uint64_t value;
	int pagemap_fd;

	if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
			  false, true, false))
		err("register failed");

	pagemap_fd = pagemap_open();

	/* Test applying pte marker to anon unpopulated */
	wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
	pagemap_check_wp(value, true);

	/* Test unprotect on anon pte marker */
	wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false);
	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
	pagemap_check_wp(value, false);

	/* Test zap on anon marker */
	wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
	if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
		err("madvise(MADV_DONTNEED) failed");
	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
	pagemap_check_wp(value, false);

	/* Test fault in after marker removed */
	*gopts->area_dst = 1;
	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
	pagemap_check_wp(value, false);
	/* Drop it to make pte none again */
	if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
		err("madvise(MADV_DONTNEED) failed");

	/* Test read-zero-page upon pte marker */
	wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
	*(volatile char *)gopts->area_dst;
	/* Drop it to make pte none again */
	if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
		err("madvise(MADV_DONTNEED) failed");

	uffd_test_pass();
}

static void uffd_wp_fork_test_common(uffd_global_test_opts_t *gopts, uffd_test_args_t *args,
				     bool with_event)
{
	int pagemap_fd;
	uint64_t value;

	if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
			  false, true, false))
		err("register failed");

	pagemap_fd = pagemap_open();

	/* Touch the page */
	*gopts->area_dst = 1;
	wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
	pagemap_check_wp(value, true);
	if (pagemap_test_fork(gopts, with_event, false)) {
		uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
			       with_event ? "missing" : "stall");
		goto out;
	}

	/*
	 * This is an attempt for zapping the pgtable so as to test the
	 * markers.
	 *
	 * For private mappings, PAGEOUT will only work on exclusive ptes
	 * (PM_MMAP_EXCLUSIVE) which we should satisfy.
	 *
	 * For shared, PAGEOUT may not work.  Use DONTNEED instead which
	 * plays a similar role of zapping (rather than freeing the page)
	 * to expose pte markers.
	 */
	if (args->mem_type->shared) {
		if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
			err("MADV_DONTNEED");
	} else {
		/*
		 * NOTE: ignore retval because private-hugetlb doesn't yet
		 * support swapping, so it could fail.
		 */
		madvise(gopts->area_dst, gopts->page_size, MADV_PAGEOUT);
	}

	/* Uffd-wp should persist even swapped out */
	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
	pagemap_check_wp(value, true);
	if (pagemap_test_fork(gopts, with_event, false)) {
		uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
			       with_event ? "missing" : "stall");
		goto out;
	}

	/* Unprotect; this tests swap pte modifications */
	wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false);
	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
	pagemap_check_wp(value, false);

	/* Fault in the page from disk */
	*gopts->area_dst = 2;
	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
	pagemap_check_wp(value, false);
	uffd_test_pass();
out:
	if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size))
		err("unregister failed");
	close(pagemap_fd);
}

static void uffd_wp_fork_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_wp_fork_test_common(gopts, args, false);
}

static void uffd_wp_fork_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_wp_fork_test_common(gopts, args, true);
}

static void uffd_wp_fork_pin_test_common(uffd_global_test_opts_t *gopts,
					 uffd_test_args_t *args,
					 bool with_event)
{
	int pagemap_fd;
	pin_args pin_args = {};

	if (uffd_register(gopts->uffd, gopts->area_dst, gopts->page_size, false, true, false))
		err("register failed");

	pagemap_fd = pagemap_open();

	/* Touch the page */
	*gopts->area_dst = 1;
	wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);

	/*
	 * 1. First pin, then fork().  This tests fork() special path when
	 * doing early CoW if the page is private.
	 */
	if (pin_pages(&pin_args, gopts->area_dst, gopts->page_size)) {
		uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
			       "or unprivileged");
		close(pagemap_fd);
		uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size);
		return;
	}

	if (pagemap_test_fork(gopts, with_event, false)) {
		uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
			       with_event ? "missing" : "stall");
		unpin_pages(&pin_args);
		goto out;
	}

	unpin_pages(&pin_args);

	/*
	 * 2. First fork(), then pin (in the child, where test_pin==true).
	 * This tests COR, aka, page unsharing on private memories.
	 */
	if (pagemap_test_fork(gopts, with_event, true)) {
		uffd_test_fail("Detected %s uffd-wp bit when RO pin",
			       with_event ? "missing" : "stall");
		goto out;
	}
	uffd_test_pass();
out:
	if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
		err("register failed");
	close(pagemap_fd);
}

static void uffd_wp_fork_pin_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_wp_fork_pin_test_common(gopts, args, false);
}

static void uffd_wp_fork_pin_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_wp_fork_pin_test_common(gopts, args, true);
}

static void check_memory_contents(uffd_global_test_opts_t *gopts, char *p)
{
	unsigned long i, j;
	uint8_t expected_byte;

	for (i = 0; i < gopts->nr_pages; ++i) {
		expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
		for (j = 0; j < gopts->page_size; j++) {
			uint8_t v = *(uint8_t *)(p + (i * gopts->page_size) + j);
			if (v != expected_byte)
				err("unexpected page contents");
		}
	}
}

static void uffd_minor_test_common(uffd_global_test_opts_t *gopts, bool test_collapse, bool test_wp)
{
	unsigned long p;
	pthread_t uffd_mon;
	char c = '\0';
	struct uffd_args args = { 0 };
	args.gopts = gopts;

	/*
	 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
	 * both do not make much sense.
	 */
	assert(!(test_collapse && test_wp));

	if (uffd_register(gopts->uffd, gopts->area_dst_alias, gopts->nr_pages * gopts->page_size,
			  /* NOTE! MADV_COLLAPSE may not work with uffd-wp */
			  false, test_wp, true))
		err("register failure");

	/*
	 * After registering with UFFD, populate the non-UFFD-registered side of
	 * the shared mapping. This should *not* trigger any UFFD minor faults.
	 */
	for (p = 0; p < gopts->nr_pages; ++p)
		memset(gopts->area_dst + (p * gopts->page_size), p % ((uint8_t)-1),
		       gopts->page_size);

	args.apply_wp = test_wp;
	if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
		err("uffd_poll_thread create");

	/*
	 * Read each of the pages back using the UFFD-registered mapping. We
	 * expect that the first time we touch a page, it will result in a minor
	 * fault. uffd_poll_thread will resolve the fault by bit-flipping the
	 * page's contents, and then issuing a CONTINUE ioctl.
	 */
	check_memory_contents(gopts, gopts->area_dst_alias);

	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
		err("pipe write");
	if (pthread_join(uffd_mon, NULL))
		err("join() failed");

	if (test_collapse) {
		if (madvise(gopts->area_dst_alias, gopts->nr_pages * gopts->page_size,
			    MADV_COLLAPSE)) {
			/* It's fine to fail for this one... */
			uffd_test_skip("MADV_COLLAPSE failed");
			return;
		}

		uffd_test_ops->check_pmd_mapping(gopts,
						 gopts->area_dst,
						 gopts->nr_pages * gopts->page_size /
						 read_pmd_pagesize());
		/*
		 * This won't cause uffd-fault - it purely just makes sure there
		 * was no corruption.
		 */
		check_memory_contents(gopts, gopts->area_dst_alias);
	}

	if (args.missing_faults != 0 || args.minor_faults != gopts->nr_pages)
		uffd_test_fail("stats check error");
	else
		uffd_test_pass();
}

void uffd_minor_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_minor_test_common(gopts, false, false);
}

void uffd_minor_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_minor_test_common(gopts, false, true);
}

void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_minor_test_common(gopts, true, false);
}

static sigjmp_buf jbuf, *sigbuf;

static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
{
	if (sig == SIGBUS) {
		if (sigbuf)
			siglongjmp(*sigbuf, 1);
		abort();
	}
}

/*
 * For non-cooperative userfaultfd test we fork() a process that will
 * generate pagefaults, will mremap the area monitored by the
 * userfaultfd and at last this process will release the monitored
 * area.
 * For the anonymous and shared memory the area is divided into two
 * parts, the first part is accessed before mremap, and the second
 * part is accessed after mremap. Since hugetlbfs does not support
 * mremap, the entire monitored area is accessed in a single pass for
 * HUGETLB_TEST.
 * The release of the pages currently generates event for shmem and
 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
 * for hugetlb.
 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
 * monitored area, generate pagefaults and test that signal is delivered.
 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
 * test robustness use case - we release monitored area, fork a process
 * that will generate pagefaults and verify signal is generated.
 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
 * feature. Using monitor thread, verify no userfault events are generated.
 */
static int faulting_process(uffd_global_test_opts_t *gopts, int signal_test, bool wp)
{
	unsigned long nr, i;
	unsigned long long count;
	unsigned long split_nr_pages;
	unsigned long lastnr;
	struct sigaction act;
	volatile unsigned long signalled = 0;

	split_nr_pages = (gopts->nr_pages + 1) / 2;

	if (signal_test) {
		sigbuf = &jbuf;
		memset(&act, 0, sizeof(act));
		act.sa_sigaction = sighndl;
		act.sa_flags = SA_SIGINFO;
		if (sigaction(SIGBUS, &act, 0))
			err("sigaction");
		lastnr = (unsigned long)-1;
	}

	for (nr = 0; nr < split_nr_pages; nr++) {
		volatile int steps = 1;
		unsigned long offset = nr * gopts->page_size;

		if (signal_test) {
			if (sigsetjmp(*sigbuf, 1) != 0) {
				if (steps == 1 && nr == lastnr)
					err("Signal repeated");

				lastnr = nr;
				if (signal_test == 1) {
					if (steps == 1) {
						/* This is a MISSING request */
						steps++;
						if (copy_page(gopts, offset, wp))
							signalled++;
					} else {
						/* This is a WP request */
						assert(steps == 2);
						wp_range(gopts->uffd,
							 (__u64)gopts->area_dst +
							 offset,
							 gopts->page_size, false);
					}
				} else {
					signalled++;
					continue;
				}
			}
		}

		count = *area_count(gopts->area_dst, nr, gopts);
		if (count != gopts->count_verify[nr])
			err("nr %lu memory corruption %llu %llu\n",
			    nr, count, gopts->count_verify[nr]);
		/*
		 * Trigger write protection if there is by writing
		 * the same value back.
		 */
		*area_count(gopts->area_dst, nr, gopts) = count;
	}

	if (signal_test)
		return signalled != split_nr_pages;

	gopts->area_dst = mremap(gopts->area_dst, gopts->nr_pages * gopts->page_size,
				 gopts->nr_pages * gopts->page_size,
				 MREMAP_MAYMOVE | MREMAP_FIXED,
				 gopts->area_src);
	if (gopts->area_dst == MAP_FAILED)
		err("mremap");
	/* Reset area_src since we just clobbered it */
	gopts->area_src = NULL;

	for (; nr < gopts->nr_pages; nr++) {
		count = *area_count(gopts->area_dst, nr, gopts);
		if (count != gopts->count_verify[nr]) {
			err("nr %lu memory corruption %llu %llu\n",
			    nr, count, gopts->count_verify[nr]);
		}
		/*
		 * Trigger write protection if there is by writing
		 * the same value back.
		 */
		*area_count(gopts->area_dst, nr, gopts) = count;
	}

	uffd_test_ops->release_pages(gopts, gopts->area_dst);

	for (nr = 0; nr < gopts->nr_pages; nr++)
		for (i = 0; i < gopts->page_size; i++)
			if (*(gopts->area_dst + nr * gopts->page_size + i) != 0)
				err("page %lu offset %lu is not zero", nr, i);

	return 0;
}

static void uffd_sigbus_test_common(uffd_global_test_opts_t *gopts, bool wp)
{
	unsigned long userfaults;
	pthread_t uffd_mon;
	pid_t pid;
	int err;
	char c = '\0';
	struct uffd_args args = { 0 };
	args.gopts = gopts;

	gopts->ready_for_fork = false;

	fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);

	if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
			  true, wp, false))
		err("register failure");

	if (faulting_process(gopts, 1, wp))
		err("faulting process failed");

	uffd_test_ops->release_pages(gopts, gopts->area_dst);

	args.apply_wp = wp;
	if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
		err("uffd_poll_thread create");

	while (!gopts->ready_for_fork)
		; /* Wait for the poll_thread to start executing before forking */

	pid = fork();
	if (pid < 0)
		err("fork");

	if (!pid)
		_exit(faulting_process(gopts, 2, wp));

	waitpid(pid, &err, 0);
	if (err)
		err("faulting process failed");
	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
		err("pipe write");
	if (pthread_join(uffd_mon, (void **)&userfaults))
		err("pthread_join()");

	if (userfaults)
		uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
	else
		uffd_test_pass();
}

static void uffd_sigbus_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_sigbus_test_common(gopts, false);
}

static void uffd_sigbus_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_sigbus_test_common(gopts, true);
}

static void uffd_events_test_common(uffd_global_test_opts_t *gopts, bool wp)
{
	pthread_t uffd_mon;
	pid_t pid;
	int err;
	char c = '\0';
	struct uffd_args args = { 0 };
	args.gopts = gopts;

	gopts->ready_for_fork = false;

	fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);
	if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
			  true, wp, false))
		err("register failure");

	args.apply_wp = wp;
	if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
		err("uffd_poll_thread create");

	while (!gopts->ready_for_fork)
		; /* Wait for the poll_thread to start executing before forking */

	pid = fork();
	if (pid < 0)
		err("fork");

	if (!pid)
		_exit(faulting_process(gopts, 0, wp));

	waitpid(pid, &err, 0);
	if (err)
		err("faulting process failed");
	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
		err("pipe write");
	if (pthread_join(uffd_mon, NULL))
		err("pthread_join()");

	if (args.missing_faults != gopts->nr_pages)
		uffd_test_fail("Fault counts wrong");
	else
		uffd_test_pass();
}

static void uffd_events_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_events_test_common(gopts, false);
}

static void uffd_events_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	uffd_events_test_common(gopts, true);
}

static void retry_uffdio_zeropage(uffd_global_test_opts_t *gopts,
				  struct uffdio_zeropage *uffdio_zeropage)
{
	uffd_test_ops->alias_mapping(gopts, &uffdio_zeropage->range.start,
				     uffdio_zeropage->range.len,
				     0);
	if (ioctl(gopts->uffd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
		if (uffdio_zeropage->zeropage != -EEXIST)
			err("UFFDIO_ZEROPAGE error: %"PRId64,
			    (int64_t)uffdio_zeropage->zeropage);
	} else {
		err("UFFDIO_ZEROPAGE error: %"PRId64,
		    (int64_t)uffdio_zeropage->zeropage);
	}
}

static bool do_uffdio_zeropage(uffd_global_test_opts_t *gopts, bool has_zeropage)
{
	struct uffdio_zeropage uffdio_zeropage = { 0 };
	int ret;
	__s64 res;

	uffdio_zeropage.range.start = (unsigned long) gopts->area_dst;
	uffdio_zeropage.range.len = gopts->page_size;
	uffdio_zeropage.mode = 0;
	ret = ioctl(gopts->uffd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
	res = uffdio_zeropage.zeropage;
	if (ret) {
		/* real retval in ufdio_zeropage.zeropage */
		if (has_zeropage)
			err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
		else if (res != -EINVAL)
			err("UFFDIO_ZEROPAGE not -EINVAL");
	} else if (has_zeropage) {
		if (res != gopts->page_size)
			err("UFFDIO_ZEROPAGE unexpected size");
		else
			retry_uffdio_zeropage(gopts, &uffdio_zeropage);
		return true;
	} else
		err("UFFDIO_ZEROPAGE succeeded");

	return false;
}

/*
 * Registers a range with MISSING mode only for zeropage test.  Return true
 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
 * because we want to detect .ioctls along the way.
 */
static bool
uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
{
	uint64_t ioctls = 0;

	if (uffd_register_with_ioctls(uffd, addr, len, true,
				      false, false, &ioctls))
		err("zeropage register fail");

	return ioctls & (1 << _UFFDIO_ZEROPAGE);
}

/* exercise UFFDIO_ZEROPAGE */
static void uffd_zeropage_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	bool has_zeropage;
	int i;

	has_zeropage = uffd_register_detect_zeropage(gopts->uffd,
						     gopts->area_dst,
						     gopts->page_size);
	if (gopts->area_dst_alias)
		/* Ignore the retval; we already have it */
		uffd_register_detect_zeropage(gopts->uffd, gopts->area_dst_alias, gopts->page_size);

	if (do_uffdio_zeropage(gopts, has_zeropage))
		for (i = 0; i < gopts->page_size; i++)
			if (gopts->area_dst[i] != 0)
				err("data non-zero at offset %d\n", i);

	if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
		err("unregister");

	if (gopts->area_dst_alias && uffd_unregister(gopts->uffd,
						     gopts->area_dst_alias,
						     gopts->page_size))
		err("unregister");

	uffd_test_pass();
}

static void uffd_register_poison(int uffd, void *addr, uint64_t len)
{
	uint64_t ioctls = 0;
	uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);

	if (uffd_register_with_ioctls(uffd, addr, len, true,
				      false, false, &ioctls))
		err("poison register fail");

	if ((ioctls & expected) != expected)
		err("registered area doesn't support COPY and POISON ioctls");
}

static void do_uffdio_poison(uffd_global_test_opts_t *gopts, unsigned long offset)
{
	struct uffdio_poison uffdio_poison = { 0 };
	int ret;
	__s64 res;

	uffdio_poison.range.start = (unsigned long) gopts->area_dst + offset;
	uffdio_poison.range.len = gopts->page_size;
	uffdio_poison.mode = 0;
	ret = ioctl(gopts->uffd, UFFDIO_POISON, &uffdio_poison);
	res = uffdio_poison.updated;

	if (ret)
		err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
	else if (res != gopts->page_size)
		err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
}

static void uffd_poison_handle_fault(uffd_global_test_opts_t *gopts,
				     struct uffd_msg *msg,
				     struct uffd_args *args)
{
	unsigned long offset;

	if (msg->event != UFFD_EVENT_PAGEFAULT)
		err("unexpected msg event %u", msg->event);

	if (msg->arg.pagefault.flags &
	    (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
		err("unexpected fault type %llu", msg->arg.pagefault.flags);

	offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst;
	offset &= ~(gopts->page_size-1);

	/* Odd pages -> copy zeroed page; even pages -> poison. */
	if (offset & gopts->page_size)
		copy_page(gopts, offset, false);
	else
		do_uffdio_poison(gopts, offset);
}

/* Make sure to cover odd/even, and minimum duplications */
#define  UFFD_POISON_TEST_NPAGES  4

static void uffd_poison_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
	pthread_t uffd_mon;
	char c;
	struct uffd_args args = { 0 };
	struct sigaction act = { 0 };
	unsigned long nr_sigbus = 0;
	unsigned long nr, poison_pages = UFFD_POISON_TEST_NPAGES;

	if (gopts->nr_pages < poison_pages) {
		uffd_test_skip("Too less pages for POISON test");
		return;
	}

	args.gopts = gopts;

	fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);

	uffd_register_poison(gopts->uffd, gopts->area_dst, poison_pages * gopts->page_size);
	memset(gopts->area_src, 0, poison_pages * gopts->page_size);

	args.handle_fault = uffd_poison_handle_fault;
	if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
		err("uffd_poll_thread create");

	sigbuf = &jbuf;
	act.sa_sigaction = sighndl;
	act.sa_flags = SA_SIGINFO;
	if (sigaction(SIGBUS, &act, 0))
		err("sigaction");

	for (nr = 0; nr < poison_pages; ++nr) {
		unsigned long offset = nr * gopts->page_size;
		const char *bytes = (const char *) gopts->area_dst + offset;
		const char *i;

		if (sigsetjmp(*sigbuf, 1)) {
			/*
			 * Access below triggered a SIGBUS, which was caught by
			 * sighndl, which then jumped here. Count this SIGBUS,
			 * and move on to next page.
			 */
			++nr_sigbus;
			continue;
		}

		for (i = bytes; i < bytes + gopts->page_size; ++i) {
			if (*i)
				err("nonzero byte in area_dst (%p) at %p: %u",
				    gopts->area_dst, i, *i);
		}
	}

	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
		err("pipe write");
	if (pthread_join(uffd_mon, NULL))
		err("pthread_join()");

	if (nr_sigbus != poison_pages / 2)
		err("expected to receive %lu SIGBUS, actually received %lu",
		    poison_pages / 2, nr_sigbus);

	uffd_test_pass();
}

static void
uffd_move_handle_fault_common(uffd_global_test_opts_t *gopts,
			      struct uffd_msg *msg,
			      struct uffd_args *args,
			      unsigned long len)
{
	unsigned long offset;

	if (msg->event != UFFD_EVENT_PAGEFAULT)
		err("unexpected msg event %u", msg->event);

	if (msg->arg.pagefault.flags &
	    (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
		err("unexpected fault type %llu", msg->arg.pagefault.flags);

	offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst;
	offset &= ~(len-1);

	if (move_page(gopts, offset, len))
		args->missing_faults++;
}

static void uffd_move_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg,
				   struct uffd_args *args)
{
	uffd_move_handle_fault_common(gopts, msg, args, gopts->page_size);
}

static void uffd_move_pmd_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg,
				       struct uffd_args *args)
{
	uffd_move_handle_fault_common(gopts, msg, args, read_pmd_pagesize());
}

static void
uffd_move_test_common(uffd_global_test_opts_t *gopts,
		      uffd_test_args_t *targs,
		      unsigned long chunk_size,
		      void (*handle_fault)(struct uffd_global_test_opts *gopts,
		      struct uffd_msg *msg, struct uffd_args *args)
)
{
	unsigned long nr;
	pthread_t uffd_mon;
	char c = '\0';
	unsigned long long count;
	struct uffd_args args = { 0 };
	char *orig_area_src = NULL, *orig_area_dst = NULL;
	unsigned long step_size, step_count;
	unsigned long src_offs = 0;
	unsigned long dst_offs = 0;

	args.gopts = gopts;

	/* Prevent source pages from being mapped more than once */
	if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_DONTFORK))
		err("madvise(MADV_DONTFORK) failure");

	if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
			  true, false, false))
		err("register failure");

	args.handle_fault = handle_fault;
	if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
		err("uffd_poll_thread create");

	step_size = chunk_size / gopts->page_size;
	step_count = gopts->nr_pages / step_size;

	if (chunk_size > gopts->page_size) {
		char *aligned_src = ALIGN_UP(gopts->area_src, chunk_size);
		char *aligned_dst = ALIGN_UP(gopts->area_dst, chunk_size);

		if (aligned_src != gopts->area_src || aligned_dst != gopts->area_dst) {
			src_offs = (aligned_src - gopts->area_src) / gopts->page_size;
			dst_offs = (aligned_dst - gopts->area_dst) / gopts->page_size;
			step_count--;
		}
		orig_area_src = gopts->area_src;
		orig_area_dst = gopts->area_dst;
		gopts->area_src = aligned_src;
		gopts->area_dst = aligned_dst;
	}

	/*
	 * Read each of the pages back using the UFFD-registered mapping. We
	 * expect that the first time we touch a page, it will result in a missing
	 * fault. uffd_poll_thread will resolve the fault by moving source
	 * page to destination.
	 */
	for (nr = 0; nr < step_count * step_size; nr += step_size) {
		unsigned long i;

		/* Check area_src content */
		for (i = 0; i < step_size; i++) {
			count = *area_count(gopts->area_src, nr + i, gopts);
			if (count != gopts->count_verify[src_offs + nr + i])
				err("nr %lu source memory invalid %llu %llu\n",
				    nr + i, count, gopts->count_verify[src_offs + nr + i]);
		}

		/* Faulting into area_dst should move the page or the huge page */
		for (i = 0; i < step_size; i++) {
			count = *area_count(gopts->area_dst, nr + i, gopts);
			if (count != gopts->count_verify[dst_offs + nr + i])
				err("nr %lu memory corruption %llu %llu\n",
				    nr, count, gopts->count_verify[dst_offs + nr + i]);
		}

		/* Re-check area_src content which should be empty */
		for (i = 0; i < step_size; i++) {
			count = *area_count(gopts->area_src, nr + i, gopts);
			if (count != 0)
				err("nr %lu move failed %llu %llu\n",
				    nr, count, gopts->count_verify[src_offs + nr + i]);
		}
	}
	if (chunk_size > gopts->page_size) {
		gopts->area_src = orig_area_src;
		gopts->area_dst = orig_area_dst;
	}

	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
		err("pipe write");
	if (pthread_join(uffd_mon, NULL))
		err("join() failed");

	if (args.missing_faults != step_count || args.minor_faults != 0)
		uffd_test_fail("stats check error");
	else
		uffd_test_pass();
}

static void uffd_move_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
	uffd_move_test_common(gopts, targs, gopts->page_size, uffd_move_handle_fault);
}

static void uffd_move_pmd_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
	if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE))
		err("madvise(MADV_HUGEPAGE) failure");
	uffd_move_test_common(gopts, targs, read_pmd_pagesize(),
			      uffd_move_pmd_handle_fault);
}

static void uffd_move_pmd_split_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
	if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE))
		err("madvise(MADV_NOHUGEPAGE) failure");
	uffd_move_test_common(gopts, targs, read_pmd_pagesize(),
			      uffd_move_pmd_handle_fault);
}

static bool
uffdio_verify_results(const char *name, int ret, int error, long result)
{
	/*
	 * Should always return -1 with errno=EAGAIN, with corresponding
	 * result field updated in ioctl() args to be -EAGAIN too
	 * (e.g. copy.copy field for UFFDIO_COPY).
	 */
	if (ret != -1) {
		uffd_test_fail("%s should have returned -1", name);
		return false;
	}

	if (error != EAGAIN) {
		uffd_test_fail("%s should have errno==EAGAIN", name);
		return false;
	}

	if (result != -EAGAIN) {
		uffd_test_fail("%s should have been updated for -EAGAIN",
			       name);
		return false;
	}

	return true;
}

/*
 * This defines a function to test one ioctl.  Note that here "field" can
 * be 1 or anything not -EAGAIN.  With that initial value set, we can
 * verify later that it should be updated by kernel (when -EAGAIN
 * returned), by checking whether it is also updated to -EAGAIN.
 */
#define DEFINE_MMAP_CHANGING_TEST(name, ioctl_name, field)		\
	static bool uffdio_mmap_changing_test_##name(int fd)		\
	{								\
		int ret;						\
		struct uffdio_##name args = {				\
			.field = 1,					\
		};							\
		ret = ioctl(fd, ioctl_name, &args);			\
		return uffdio_verify_results(#ioctl_name, ret, errno, args.field); \
	}

DEFINE_MMAP_CHANGING_TEST(zeropage, UFFDIO_ZEROPAGE, zeropage)
DEFINE_MMAP_CHANGING_TEST(copy, UFFDIO_COPY, copy)
DEFINE_MMAP_CHANGING_TEST(move, UFFDIO_MOVE, move)
DEFINE_MMAP_CHANGING_TEST(poison, UFFDIO_POISON, updated)
DEFINE_MMAP_CHANGING_TEST(continue, UFFDIO_CONTINUE, mapped)

typedef enum {
	/* We actually do not care about any state except UNINTERRUPTIBLE.. */
	THR_STATE_UNKNOWN = 0,
	THR_STATE_UNINTERRUPTIBLE,
} thread_state;

typedef struct {
	uffd_global_test_opts_t *gopts;
	volatile pid_t *pid;
} mmap_changing_thread_args;

static void sleep_short(void)
{
	usleep(1000);
}

static thread_state thread_state_get(pid_t tid)
{
	const char *header = "State:\t";
	char tmp[256], *p, c;
	FILE *fp;

	snprintf(tmp, sizeof(tmp), "/proc/%d/status", tid);
	fp = fopen(tmp, "r");

	if (!fp)
		return THR_STATE_UNKNOWN;

	while (fgets(tmp, sizeof(tmp), fp)) {
		p = strstr(tmp, header);
		if (p) {
			/* For example, "State:\tD (disk sleep)" */
			c = *(p + strlen(header));
			return c == 'D' ?
			    THR_STATE_UNINTERRUPTIBLE : THR_STATE_UNKNOWN;
		}
	}

	return THR_STATE_UNKNOWN;
}

static void thread_state_until(pid_t tid, thread_state state)
{
	thread_state s;

	do {
		s = thread_state_get(tid);
		sleep_short();
	} while (s != state);
}

static void *uffd_mmap_changing_thread(void *opaque)
{
	mmap_changing_thread_args *args = opaque;
	uffd_global_test_opts_t *gopts = args->gopts;
	volatile pid_t *pid = args->pid;
	int ret;

	/* Unfortunately, it's only fetch-able from the thread itself.. */
	assert(*pid == 0);
	*pid = syscall(SYS_gettid);

	/* Inject an event, this will hang solid until the event read */
	ret = madvise(gopts->area_dst, gopts->page_size, MADV_REMOVE);
	if (ret)
		err("madvise(MADV_REMOVE) failed");

	return NULL;
}

static void uffd_consume_message(uffd_global_test_opts_t *gopts)
{
	struct uffd_msg msg = { 0 };

	while (uffd_read_msg(gopts, &msg));
}

static void uffd_mmap_changing_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
	/*
	 * This stores the real PID (which can be different from how tid is
	 * defined..) for the child thread, 0 means not initialized.
	 */
	pid_t pid = 0;
	pthread_t tid;
	int ret;
	mmap_changing_thread_args args = { gopts, &pid };

	if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
			  true, false, false))
		err("uffd_register() failed");

	/* Create a thread to generate the racy event */
	ret = pthread_create(&tid, NULL, uffd_mmap_changing_thread, &args);
	if (ret)
		err("pthread_create() failed");

	/*
	 * Wait until the thread setup the pid.  Use volatile to make sure
	 * it reads from RAM not regs.
	 */
	while (!(volatile pid_t)pid)
		sleep_short();

	/* Wait until the thread hangs at REMOVE event */
	thread_state_until(pid, THR_STATE_UNINTERRUPTIBLE);

	if (!uffdio_mmap_changing_test_copy(gopts->uffd))
		return;

	if (!uffdio_mmap_changing_test_zeropage(gopts->uffd))
		return;

	if (!uffdio_mmap_changing_test_move(gopts->uffd))
		return;

	if (!uffdio_mmap_changing_test_poison(gopts->uffd))
		return;

	if (!uffdio_mmap_changing_test_continue(gopts->uffd))
		return;

	/*
	 * All succeeded above!  Recycle everything.  Start by reading the
	 * event so as to kick the thread roll again..
	 */
	uffd_consume_message(gopts);

	ret = pthread_join(tid, NULL);
	assert(ret == 0);

	uffd_test_pass();
}

static int prevent_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg)
{
	/* This should be done before source area is populated */
	if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE)) {
		/* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
		if (errno != EINVAL) {
			if (errmsg)
				*errmsg = "madvise(MADV_NOHUGEPAGE) failed";
			return -errno;
		}
	}
	return 0;
}

static int request_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg)
{
	/* This should be done before source area is populated */
	if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE)) {
		if (errmsg) {
			*errmsg = (errno == EINVAL) ?
				"CONFIG_TRANSPARENT_HUGEPAGE is not set" :
				"madvise(MADV_HUGEPAGE) failed";
		}
		return -errno;
	}
	return 0;
}

struct uffd_test_case_ops uffd_move_test_case_ops = {
	.post_alloc = prevent_hugepages,
};

struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
	.post_alloc = request_hugepages,
};

/*
 * Test the returned uffdio_register.ioctls with different register modes.
 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
 */
static void
do_register_ioctls_test(uffd_global_test_opts_t *gopts,
			uffd_test_args_t *args,
			bool miss,
			bool wp,
			bool minor)
{
	uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
	mem_type_t *mem_type = args->mem_type;
	int ret;

	ret = uffd_register_with_ioctls(gopts->uffd, gopts->area_dst, gopts->page_size,
					miss, wp, minor, &ioctls);

	/*
	 * Handle special cases of UFFDIO_REGISTER here where it should
	 * just fail with -EINVAL first..
	 *
	 * Case 1: register MINOR on anon
	 * Case 2: register with no mode selected
	 */
	if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
	    (!miss && !wp && !minor)) {
		if (ret != -EINVAL)
			err("register (miss=%d, wp=%d, minor=%d) failed "
			    "with wrong errno=%d", miss, wp, minor, ret);
		return;
	}

	/* UFFDIO_REGISTER should succeed, then check ioctls returned */
	if (miss)
		expected |= BIT_ULL(_UFFDIO_COPY);
	if (wp)
		expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
	if (minor)
		expected |= BIT_ULL(_UFFDIO_CONTINUE);

	if ((ioctls & expected) != expected)
		err("unexpected uffdio_register.ioctls "
		    "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
		    "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);

	if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
		err("unregister");
}

static void uffd_register_ioctls_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
	int miss, wp, minor;

	for (miss = 0; miss <= 1; miss++)
		for (wp = 0; wp <= 1; wp++)
			for (minor = 0; minor <= 1; minor++)
				do_register_ioctls_test(gopts, args, miss, wp, minor);

	uffd_test_pass();
}

uffd_test_case_t uffd_tests[] = {
	{
		/* Test returned uffdio_register.ioctls. */
		.name = "register-ioctls",
		.uffd_fn = uffd_register_ioctls_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
		UFFD_FEATURE_MISSING_SHMEM |
		UFFD_FEATURE_PAGEFAULT_FLAG_WP |
		UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
		UFFD_FEATURE_MINOR_HUGETLBFS |
		UFFD_FEATURE_MINOR_SHMEM,
	},
	{
		.name = "zeropage",
		.uffd_fn = uffd_zeropage_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = 0,
	},
	{
		.name = "move",
		.uffd_fn = uffd_move_test,
		.mem_targets = MEM_ANON,
		.uffd_feature_required = UFFD_FEATURE_MOVE,
		.test_case_ops = &uffd_move_test_case_ops,
	},
	{
		.name = "move-pmd",
		.uffd_fn = uffd_move_pmd_test,
		.mem_targets = MEM_ANON,
		.uffd_feature_required = UFFD_FEATURE_MOVE,
		.test_case_ops = &uffd_move_test_pmd_case_ops,
	},
	{
		.name = "move-pmd-split",
		.uffd_fn = uffd_move_pmd_split_test,
		.mem_targets = MEM_ANON,
		.uffd_feature_required = UFFD_FEATURE_MOVE,
		.test_case_ops = &uffd_move_test_pmd_case_ops,
	},
	{
		.name = "wp-fork",
		.uffd_fn = uffd_wp_fork_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
		UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
	},
	{
		.name = "wp-fork-with-event",
		.uffd_fn = uffd_wp_fork_with_event_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
		UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
		/* when set, child process should inherit uffd-wp bits */
		UFFD_FEATURE_EVENT_FORK,
	},
	{
		.name = "wp-fork-pin",
		.uffd_fn = uffd_wp_fork_pin_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
		UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
	},
	{
		.name = "wp-fork-pin-with-event",
		.uffd_fn = uffd_wp_fork_pin_with_event_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
		UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
		/* when set, child process should inherit uffd-wp bits */
		UFFD_FEATURE_EVENT_FORK,
	},
	{
		.name = "wp-unpopulated",
		.uffd_fn = uffd_wp_unpopulated_test,
		.mem_targets = MEM_ANON,
		.uffd_feature_required =
		UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
	},
	{
		.name = "minor",
		.uffd_fn = uffd_minor_test,
		.mem_targets = MEM_SHMEM | MEM_HUGETLB,
		.uffd_feature_required =
		UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
	},
	{
		.name = "minor-wp",
		.uffd_fn = uffd_minor_wp_test,
		.mem_targets = MEM_SHMEM | MEM_HUGETLB,
		.uffd_feature_required =
		UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
		UFFD_FEATURE_PAGEFAULT_FLAG_WP |
		/*
		 * HACK: here we leveraged WP_UNPOPULATED to detect whether
		 * minor mode supports wr-protect.  There's no feature flag
		 * for it so this is the best we can test against.
		 */
		UFFD_FEATURE_WP_UNPOPULATED,
	},
	{
		.name = "minor-collapse",
		.uffd_fn = uffd_minor_collapse_test,
		/* MADV_COLLAPSE only works with shmem */
		.mem_targets = MEM_SHMEM,
		/* We can't test MADV_COLLAPSE, so try our luck */
		.uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
	},
	{
		.name = "sigbus",
		.uffd_fn = uffd_sigbus_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_SIGBUS |
		UFFD_FEATURE_EVENT_FORK,
	},
	{
		.name = "sigbus-wp",
		.uffd_fn = uffd_sigbus_wp_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_SIGBUS |
		UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |
		UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
	},
	{
		.name = "events",
		.uffd_fn = uffd_events_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
		UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
	},
	{
		.name = "events-wp",
		.uffd_fn = uffd_events_wp_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
		UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
		UFFD_FEATURE_PAGEFAULT_FLAG_WP |
		UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
	},
	{
		.name = "poison",
		.uffd_fn = uffd_poison_test,
		.mem_targets = MEM_ALL,
		.uffd_feature_required = UFFD_FEATURE_POISON,
	},
	{
		.name = "mmap-changing",
		.uffd_fn = uffd_mmap_changing_test,
		/*
		 * There's no point running this test over all mem types as
		 * they share the same code paths.
		 *
		 * Choose shmem for simplicity, because (1) shmem supports
		 * MINOR mode to cover UFFDIO_CONTINUE, and (2) shmem is
		 * almost always available (unlike hugetlb).  Here we
		 * abused SHMEM for UFFDIO_MOVE, but the test we want to
		 * cover doesn't yet need the correct memory type..
		 */
		.mem_targets = MEM_SHMEM,
		/*
		 * Any UFFD_FEATURE_EVENT_* should work to trigger the
		 * race logically, but choose the simplest (REMOVE).
		 *
		 * Meanwhile, since we'll cover quite a few new ioctl()s
		 * (CONTINUE, POISON, MOVE), skip this test for old kernels
		 * by choosing all of them.
		 */
		.uffd_feature_required = UFFD_FEATURE_EVENT_REMOVE |
		UFFD_FEATURE_MOVE | UFFD_FEATURE_POISON |
		UFFD_FEATURE_MINOR_SHMEM,
	},
};

static void usage(const char *prog)
{
	printf("usage: %s [-f TESTNAME]\n", prog);
	puts("");
	puts(" -f: test name to filter (e.g., event)");
	puts(" -h: show the help msg");
	puts(" -l: list tests only");
	puts("");
	exit(KSFT_FAIL);
}

static int uffd_count_tests(int n_tests, int n_mems, const char *test_filter)
{
	uffd_test_case_t *test;
	int i, j, count = 0;

	if (!test_filter)
		count += 2;	/* test_uffd_api(false) + test_uffd_api(true) */

	for (i = 0; i < n_tests; i++) {
		test = &uffd_tests[i];
		if (test_filter && !strstr(test->name, test_filter))
			continue;
		for (j = 0; j < n_mems; j++)
			if (test->mem_targets & mem_types[j].mem_flag)
				count++;
	}

	return count;
}

static unsigned long uffd_setup_hugetlb(void)
{
	unsigned long nr_hugepages, hp_size;

	hugetlb_save_settings();
	hp_size = default_huge_page_size();

	if (!hp_size)
		return 0;

	/* need twice UFFD_TEST_MEM_SIZE, one for src area and one for dst */
	nr_hugepages = 2 * MAX(UFFD_TEST_MEM_SIZE, hp_size * 2) / hp_size;
	hugetlb_set_nr_default_pages(nr_hugepages);

	if (hugetlb_free_default_pages() < nr_hugepages)
		return 0;

	return hp_size;
}

int main(int argc, char *argv[])
{
	int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
	int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
	const char *test_filter = NULL;
	unsigned long hugepage_size;
	bool list_only = false;
	uffd_test_case_t *test;
	mem_type_t *mem_type;
	uffd_test_args_t args;
	const char *errmsg;
	int i, j, opt;

	while ((opt = getopt(argc, argv, "f:hl")) != -1) {
		switch (opt) {
		case 'f':
			test_filter = optarg;
			break;
		case 'l':
			list_only = true;
			break;
		case 'h':
		default:
			/* Unknown */
			usage(argv[0]);
			break;
		}
	}

	if (list_only) {
		for (i = 0; i < n_tests; i++) {
			test = &uffd_tests[i];
			if (test_filter && !strstr(test->name, test_filter))
				continue;
			printf("%s\n", test->name);
		}
		return KSFT_PASS;
	}

	hugepage_size = uffd_setup_hugetlb();

	ksft_print_header();
	ksft_set_plan(uffd_count_tests(n_tests, n_mems, test_filter));

	if (!test_filter) {
		test_uffd_api(false);
		test_uffd_api(true);
	}

	for (i = 0; i < n_tests; i++) {
		test = &uffd_tests[i];
		if (test_filter && !strstr(test->name, test_filter))
			continue;
		for (j = 0; j < n_mems; j++) {
			mem_type = &mem_types[j];

			/* Initialize global test options */
			uffd_global_test_opts_t gopts = { 0 };

			gopts.map_shared = mem_type->shared;
			uffd_test_ops = mem_type->mem_ops;
			uffd_test_case_ops = test->test_case_ops;

			if (!(test->mem_targets & mem_type->mem_flag))
				continue;

			uffd_test_start("%s on %s", test->name, mem_type->name);
			if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) {
				gopts.page_size = hugepage_size;
				if (gopts.page_size == 0) {
					uffd_test_skip("not enough HugeTLB pages");
					continue;
				}
			} else {
				gopts.page_size = psize();
			}

			/* Ensure we have at least 2 pages */
			gopts.nr_pages = MAX(UFFD_TEST_MEM_SIZE, gopts.page_size * 2)
				/ gopts.page_size;

			gopts.nr_parallel = 1;

			/* Initialize test arguments */
			args.mem_type = mem_type;

			if (!uffd_feature_supported(test)) {
				uffd_test_skip("feature missing");
				continue;
			}
			if (uffd_test_ctx_init(&gopts, test->uffd_feature_required, &errmsg)) {
				uffd_test_skip(errmsg);
				continue;
			}
			test->uffd_fn(&gopts, &args);
			uffd_test_ctx_clear(&gopts);
		}
	}

	ksft_finished();
}

#else /* __NR_userfaultfd */

#warning "missing __NR_userfaultfd definition"

int main(void)
{
	ksft_print_header();
	ksft_exit_skip("missing __NR_userfaultfd definition\n");
}

#endif /* __NR_userfaultfd */