1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Hold a readdir of a directory in the page fault of its buffer and see 4 * whether a create and a lookup in that directory wait for it. For a 5 * directory that is permanently empty they must not. 6 */ 7 #ifndef __SELFTESTS_READDIR_HOLD_H 8 #define __SELFTESTS_READDIR_HOLD_H 9 10 #include <errno.h> 11 #include <fcntl.h> 12 #include <poll.h> 13 #include <pthread.h> 14 #include <stdbool.h> 15 #include <stdio.h> 16 #include <stdlib.h> 17 #include <string.h> 18 #include <unistd.h> 19 #include <linux/userfaultfd.h> 20 #include <sys/ioctl.h> 21 #include <sys/mman.h> 22 #include <sys/stat.h> 23 #include <sys/syscall.h> 24 25 #define HOLD_FAULT_MS 5000 /* for the reader to reach the fault */ 26 #define HOLD_QUEUE_MS 2000 /* for the create to queue up behind it */ 27 #define HOLD_LOOKUP_MS 5000 /* for the lookup to come back */ 28 29 struct readdir_hold { 30 int uffd; 31 int taskdir; /* /proc/self/task, from before any namespace change */ 32 long page_size; 33 char *page; /* faults until released */ 34 int dfd; 35 pid_t creator_tid; 36 int created; 37 int done[2]; /* the finder writes a byte when it is back */ 38 }; 39 40 /* 41 * The fault happens in the kernel, so the userfaultfd needs CAP_SYS_PTRACE 42 * in the initial user namespace or vm.unprivileged_userfaultfd. Call this 43 * before entering a user namespace. 44 */ 45 static inline int readdir_hold_init(struct readdir_hold *h) 46 { 47 struct uffdio_api api = { .api = UFFD_API }; 48 49 h->page = MAP_FAILED; 50 h->page_size = sysconf(_SC_PAGESIZE); 51 h->taskdir = open("/proc/self/task", O_RDONLY | O_DIRECTORY | O_CLOEXEC); 52 h->uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 53 if (h->uffd < 0 || h->taskdir < 0 || ioctl(h->uffd, UFFDIO_API, &api)) { 54 if (h->uffd >= 0) 55 close(h->uffd); 56 if (h->taskdir >= 0) 57 close(h->taskdir); 58 h->uffd = h->taskdir = -1; 59 return -1; 60 } 61 return 0; 62 } 63 64 static inline void readdir_hold_destroy(struct readdir_hold *h) 65 { 66 if (h->uffd >= 0) 67 close(h->uffd); 68 if (h->taskdir >= 0) 69 close(h->taskdir); 70 h->uffd = h->taskdir = -1; 71 } 72 73 static inline void *readdir_hold_reader(void *arg) 74 { 75 struct readdir_hold *h = arg; 76 77 /* the first byte written to the buffer faults until released */ 78 syscall(__NR_getdents64, h->dfd, h->page, h->page_size); 79 return NULL; 80 } 81 82 static inline void *readdir_hold_creator(void *arg) 83 { 84 struct readdir_hold *h = arg; 85 86 h->creator_tid = syscall(__NR_gettid); 87 /* takes the directory lock exclusive before it fails */ 88 mkdirat(h->dfd, "x", 0755); 89 __atomic_store_n(&h->created, 1, __ATOMIC_RELEASE); 90 return NULL; 91 } 92 93 static inline void *readdir_hold_finder(void *arg) 94 { 95 struct readdir_hold *h = arg; 96 int fd; 97 98 /* a lookup that misses the dcache takes the lock shared */ 99 fd = openat(h->dfd, "no_such_name", O_RDONLY | O_CLOEXEC); 100 if (fd >= 0) 101 close(fd); 102 if (write(h->done[1], "x", 1) != 1) 103 perror("readdir_hold: finder"); 104 return NULL; 105 } 106 107 /* the creator is back, or waits in the kernel for the lock */ 108 static inline bool readdir_hold_creator_settled(struct readdir_hold *h) 109 { 110 char path[32], buf[256], *p; 111 ssize_t n; 112 int fd; 113 114 if (__atomic_load_n(&h->created, __ATOMIC_ACQUIRE)) 115 return true; 116 if (!h->creator_tid) 117 return false; 118 snprintf(path, sizeof(path), "%d/stat", h->creator_tid); 119 fd = openat(h->taskdir, path, O_RDONLY | O_CLOEXEC); 120 if (fd < 0) 121 return false; 122 n = read(fd, buf, sizeof(buf) - 1); 123 close(fd); 124 if (n <= 0) 125 return false; 126 buf[n] = 0; 127 /* "pid (comm) state ..." */ 128 p = strrchr(buf, ')'); 129 return p && p[1] == ' ' && p[2] == 'D'; 130 } 131 132 static inline bool readdir_hold_faulted(struct readdir_hold *h) 133 { 134 struct pollfd pfd = { .fd = h->uffd, .events = POLLIN }; 135 struct uffd_msg msg; 136 137 if (poll(&pfd, 1, HOLD_FAULT_MS) != 1) 138 return false; 139 if (read(h->uffd, &msg, sizeof(msg)) != sizeof(msg)) 140 return false; 141 return msg.event == UFFD_EVENT_PAGEFAULT; 142 } 143 144 /* let the reader go on */ 145 static inline void readdir_hold_release(struct readdir_hold *h) 146 { 147 struct uffdio_copy cp = { 148 .dst = (unsigned long)h->page, 149 .len = h->page_size, 150 }; 151 void *zero; 152 153 zero = calloc(1, h->page_size); 154 if (!zero) 155 return; 156 cp.src = (unsigned long)zero; 157 if (ioctl(h->uffd, UFFDIO_COPY, &cp) && errno != EEXIST) 158 perror("readdir_hold: UFFDIO_COPY"); 159 free(zero); 160 } 161 162 /* 163 * A readdir of @dfd that sticks in the fault of its buffer, then a create 164 * and a lookup in @dfd. Whether the lookup came back while the readdir was 165 * still stuck goes to @stalled. Returns -1 when that could not be found 166 * out. 167 */ 168 static inline int readdir_hold_check(struct readdir_hold *h, int dfd, 169 bool *stalled) 170 { 171 pthread_t reader, creator, finder; 172 struct uffdio_register reg = {}; 173 struct pollfd pfd; 174 int ret = -1, i; 175 176 h->dfd = dfd; 177 h->created = 0; 178 h->creator_tid = 0; 179 h->page = mmap(NULL, h->page_size, PROT_READ | PROT_WRITE, 180 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 181 if (h->page == MAP_FAILED) 182 return -1; 183 reg.range.start = (unsigned long)h->page; 184 reg.range.len = h->page_size; 185 reg.mode = UFFDIO_REGISTER_MODE_MISSING; 186 if (ioctl(h->uffd, UFFDIO_REGISTER, ®) || pipe2(h->done, O_CLOEXEC)) 187 goto out_page; 188 189 if (pthread_create(&reader, NULL, readdir_hold_reader, h)) 190 goto out_pipe; 191 if (!readdir_hold_faulted(h)) 192 goto out_reader; 193 if (pthread_create(&creator, NULL, readdir_hold_creator, h)) 194 goto out_reader; 195 for (i = 0; i < HOLD_QUEUE_MS / 10 && !readdir_hold_creator_settled(h); i++) 196 usleep(10000); 197 if (pthread_create(&finder, NULL, readdir_hold_finder, h)) 198 goto out_creator; 199 200 pfd.fd = h->done[0]; 201 pfd.events = POLLIN; 202 *stalled = poll(&pfd, 1, HOLD_LOOKUP_MS) != 1; 203 ret = 0; 204 205 readdir_hold_release(h); 206 pthread_join(finder, NULL); 207 out_creator: 208 if (ret) 209 readdir_hold_release(h); 210 pthread_join(creator, NULL); 211 out_reader: 212 if (ret) 213 readdir_hold_release(h); 214 pthread_join(reader, NULL); 215 out_pipe: 216 close(h->done[0]); 217 close(h->done[1]); 218 out_page: 219 munmap(h->page, h->page_size); 220 h->page = MAP_FAILED; 221 return ret; 222 } 223 224 #endif /* __SELFTESTS_READDIR_HOLD_H */ 225