1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KVM userfaultfd util 4 * Adapted from demand_paging_test.c 5 * 6 * Copyright (C) 2018, Red Hat, Inc. 7 * Copyright (C) 2019-2022 Google LLC 8 */ 9 #include <inttypes.h> 10 #include <stdio.h> 11 #include <stdlib.h> 12 #include <time.h> 13 #include <poll.h> 14 #include <pthread.h> 15 #include <linux/userfaultfd.h> 16 #include <sys/epoll.h> 17 #include <sys/syscall.h> 18 19 #include "kvm_util.h" 20 #include "test_util.h" 21 #include "memstress.h" 22 #include "userfaultfd_util.h" 23 24 #ifdef __NR_userfaultfd 25 26 static void *uffd_handler_thread_fn(void *arg) 27 { 28 struct uffd_reader_args *reader_args = (struct uffd_reader_args *)arg; 29 int uffd = reader_args->uffd; 30 int64_t pages = 0; 31 struct timespec start; 32 struct timespec ts_diff; 33 struct epoll_event evt; 34 int epollfd; 35 36 epollfd = epoll_create(1); 37 TEST_ASSERT(epollfd >= 0, "Failed to create epollfd."); 38 39 evt.events = EPOLLIN | EPOLLEXCLUSIVE; 40 evt.data.u32 = 0; 41 TEST_ASSERT(!epoll_ctl(epollfd, EPOLL_CTL_ADD, uffd, &evt), 42 "Failed to add uffd to epollfd"); 43 44 evt.events = EPOLLIN; 45 evt.data.u32 = 1; 46 TEST_ASSERT(!epoll_ctl(epollfd, EPOLL_CTL_ADD, reader_args->pipe, &evt), 47 "Failed to add pipe to epollfd"); 48 49 clock_gettime(CLOCK_MONOTONIC, &start); 50 while (1) { 51 struct uffd_msg msg; 52 int r; 53 54 r = epoll_wait(epollfd, &evt, 1, -1); 55 TEST_ASSERT(r == 1, 56 "Unexpected number of events (%d) from epoll, errno = %d", 57 r, errno); 58 59 if (evt.data.u32 == 1) { 60 char tmp_chr; 61 62 TEST_ASSERT(!(evt.events & (EPOLLERR | EPOLLHUP)), 63 "Reader thread received EPOLLERR or EPOLLHUP on pipe."); 64 r = read(reader_args->pipe, &tmp_chr, 1); 65 TEST_ASSERT(r == 1, 66 "Error reading pipefd in uffd reader thread"); 67 break; 68 } 69 70 TEST_ASSERT(!(evt.events & (EPOLLERR | EPOLLHUP)), 71 "Reader thread received EPOLLERR or EPOLLHUP on uffd."); 72 73 r = read(uffd, &msg, sizeof(msg)); 74 if (r == -1) { 75 TEST_ASSERT(errno == EAGAIN, 76 "Error reading from UFFD: errno = %d", errno); 77 continue; 78 } 79 80 TEST_ASSERT(r == sizeof(msg), 81 "Read on uffd returned unexpected number of bytes (%d)", r); 82 83 if (!(msg.event & UFFD_EVENT_PAGEFAULT)) 84 continue; 85 86 if (reader_args->delay) 87 usleep(reader_args->delay); 88 r = reader_args->handler(reader_args->uffd_mode, uffd, &msg); 89 TEST_ASSERT(r >= 0, 90 "Reader thread handler fn returned negative value %d", r); 91 pages++; 92 } 93 94 ts_diff = timespec_elapsed(start); 95 PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n", 96 pages, ts_diff.tv_sec, ts_diff.tv_nsec, 97 pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / NSEC_PER_SEC)); 98 99 return NULL; 100 } 101 102 struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay, 103 void *hva, uint64_t len, 104 uint64_t num_readers, 105 uffd_handler_t handler) 106 { 107 struct uffd_desc *uffd_desc; 108 bool is_minor = (uffd_mode == UFFDIO_REGISTER_MODE_MINOR); 109 int uffd; 110 struct uffdio_api uffdio_api; 111 struct uffdio_register uffdio_register; 112 uint64_t expected_ioctls = ((uint64_t) 1) << _UFFDIO_COPY; 113 int ret, i; 114 115 PER_PAGE_DEBUG("Userfaultfd %s mode, faults resolved with %s\n", 116 is_minor ? "MINOR" : "MISSING", 117 is_minor ? "UFFDIO_CONINUE" : "UFFDIO_COPY"); 118 119 uffd_desc = malloc(sizeof(struct uffd_desc)); 120 TEST_ASSERT(uffd_desc, "Failed to malloc uffd descriptor"); 121 122 uffd_desc->pipefds = calloc(sizeof(int), num_readers); 123 TEST_ASSERT(uffd_desc->pipefds, "Failed to alloc pipes"); 124 125 uffd_desc->readers = calloc(sizeof(pthread_t), num_readers); 126 TEST_ASSERT(uffd_desc->readers, "Failed to alloc reader threads"); 127 128 uffd_desc->reader_args = calloc(sizeof(struct uffd_reader_args), num_readers); 129 TEST_ASSERT(uffd_desc->reader_args, "Failed to alloc reader_args"); 130 131 uffd_desc->num_readers = num_readers; 132 133 /* In order to get minor faults, prefault via the alias. */ 134 if (is_minor) 135 expected_ioctls = ((uint64_t) 1) << _UFFDIO_CONTINUE; 136 137 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 138 TEST_ASSERT(uffd >= 0, "uffd creation failed, errno: %d", errno); 139 140 uffdio_api.api = UFFD_API; 141 uffdio_api.features = 0; 142 TEST_ASSERT(ioctl(uffd, UFFDIO_API, &uffdio_api) != -1, 143 "ioctl UFFDIO_API failed: %" PRIu64, 144 (uint64_t)uffdio_api.api); 145 146 uffdio_register.range.start = (uint64_t)hva; 147 uffdio_register.range.len = len; 148 uffdio_register.mode = uffd_mode; 149 TEST_ASSERT(ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) != -1, 150 "ioctl UFFDIO_REGISTER failed"); 151 TEST_ASSERT((uffdio_register.ioctls & expected_ioctls) == 152 expected_ioctls, "missing userfaultfd ioctls"); 153 154 uffd_desc->uffd = uffd; 155 for (i = 0; i < uffd_desc->num_readers; ++i) { 156 int pipes[2]; 157 158 ret = pipe2((int *) &pipes, O_CLOEXEC | O_NONBLOCK); 159 TEST_ASSERT(!ret, "Failed to set up pipefd %i for uffd_desc %p", 160 i, uffd_desc); 161 162 uffd_desc->pipefds[i] = pipes[1]; 163 164 uffd_desc->reader_args[i].uffd_mode = uffd_mode; 165 uffd_desc->reader_args[i].uffd = uffd; 166 uffd_desc->reader_args[i].delay = delay; 167 uffd_desc->reader_args[i].handler = handler; 168 uffd_desc->reader_args[i].pipe = pipes[0]; 169 170 pthread_create(&uffd_desc->readers[i], NULL, uffd_handler_thread_fn, 171 &uffd_desc->reader_args[i]); 172 173 PER_VCPU_DEBUG("Created uffd thread %i for HVA range [%p, %p)\n", 174 i, hva, hva + len); 175 } 176 177 return uffd_desc; 178 } 179 180 void uffd_stop_demand_paging(struct uffd_desc *uffd) 181 { 182 char c = 0; 183 int i; 184 185 for (i = 0; i < uffd->num_readers; ++i) 186 TEST_ASSERT(write(uffd->pipefds[i], &c, 1) == 1, 187 "Unable to write to pipefd %i for uffd_desc %p", i, uffd); 188 189 for (i = 0; i < uffd->num_readers; ++i) 190 TEST_ASSERT(!pthread_join(uffd->readers[i], NULL), 191 "Pthread_join failed on reader %i for uffd_desc %p", i, uffd); 192 193 close(uffd->uffd); 194 195 for (i = 0; i < uffd->num_readers; ++i) { 196 close(uffd->pipefds[i]); 197 close(uffd->reader_args[i].pipe); 198 } 199 200 free(uffd->pipefds); 201 free(uffd->readers); 202 free(uffd->reader_args); 203 free(uffd); 204 } 205 206 #endif /* __NR_userfaultfd */ 207