1 // SPDX-License-Identifier: GPL-2.0 2 #define _GNU_SOURCE 3 #include <linux/compiler.h> 4 #include <asm/barrier.h> 5 #include <test_progs.h> 6 #include <sys/mman.h> 7 #include <sys/epoll.h> 8 #include <time.h> 9 #include <sched.h> 10 #include <signal.h> 11 #include <pthread.h> 12 #include <sys/sysinfo.h> 13 #include <linux/perf_event.h> 14 #include <linux/ring_buffer.h> 15 #include "test_ringbuf.lskel.h" 16 #include "test_ringbuf_map_key.lskel.h" 17 18 #define EDONE 7777 19 20 static int duration = 0; 21 22 struct sample { 23 int pid; 24 int seq; 25 long value; 26 char comm[16]; 27 }; 28 29 static int sample_cnt; 30 31 static void atomic_inc(int *cnt) 32 { 33 __atomic_add_fetch(cnt, 1, __ATOMIC_SEQ_CST); 34 } 35 36 static int atomic_xchg(int *cnt, int val) 37 { 38 return __atomic_exchange_n(cnt, val, __ATOMIC_SEQ_CST); 39 } 40 41 static int process_sample(void *ctx, void *data, size_t len) 42 { 43 struct sample *s = data; 44 45 atomic_inc(&sample_cnt); 46 47 switch (s->seq) { 48 case 0: 49 CHECK(s->value != 333, "sample1_value", "exp %ld, got %ld\n", 50 333L, s->value); 51 return 0; 52 case 1: 53 CHECK(s->value != 777, "sample2_value", "exp %ld, got %ld\n", 54 777L, s->value); 55 return -EDONE; 56 default: 57 /* we don't care about the rest */ 58 return 0; 59 } 60 } 61 62 static struct test_ringbuf_map_key_lskel *skel_map_key; 63 static struct test_ringbuf_lskel *skel; 64 static struct ring_buffer *ringbuf; 65 66 static void trigger_samples() 67 { 68 skel->bss->dropped = 0; 69 skel->bss->total = 0; 70 skel->bss->discarded = 0; 71 72 /* trigger exactly two samples */ 73 skel->bss->value = 333; 74 syscall(__NR_getpgid); 75 skel->bss->value = 777; 76 syscall(__NR_getpgid); 77 } 78 79 static void *poll_thread(void *input) 80 { 81 long timeout = (long)input; 82 83 return (void *)(long)ring_buffer__poll(ringbuf, timeout); 84 } 85 86 static void ringbuf_subtest(void) 87 { 88 const size_t rec_sz = BPF_RINGBUF_HDR_SZ + sizeof(struct sample); 89 pthread_t thread; 90 long bg_ret = -1; 91 int err, cnt, rb_fd; 92 int page_size = getpagesize(); 93 void *mmap_ptr, *tmp_ptr; 94 struct ring *ring; 95 int map_fd; 96 unsigned long avail_data, ring_size, cons_pos, prod_pos; 97 98 skel = test_ringbuf_lskel__open(); 99 if (CHECK(!skel, "skel_open", "skeleton open failed\n")) 100 return; 101 102 skel->maps.ringbuf.max_entries = page_size; 103 104 err = test_ringbuf_lskel__load(skel); 105 if (CHECK(err != 0, "skel_load", "skeleton load failed\n")) 106 goto cleanup; 107 108 rb_fd = skel->maps.ringbuf.map_fd; 109 /* good read/write cons_pos */ 110 mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, rb_fd, 0); 111 ASSERT_OK_PTR(mmap_ptr, "rw_cons_pos"); 112 tmp_ptr = mremap(mmap_ptr, page_size, 2 * page_size, MREMAP_MAYMOVE); 113 if (!ASSERT_ERR_PTR(tmp_ptr, "rw_extend")) 114 goto cleanup; 115 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_cons_pos_protect"); 116 ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw"); 117 118 /* bad writeable prod_pos */ 119 mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, page_size); 120 err = -errno; 121 ASSERT_ERR_PTR(mmap_ptr, "wr_prod_pos"); 122 ASSERT_EQ(err, -EPERM, "wr_prod_pos_err"); 123 124 /* bad writeable data pages */ 125 mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size); 126 err = -errno; 127 ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_one"); 128 ASSERT_EQ(err, -EPERM, "wr_data_page_one_err"); 129 mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 3 * page_size); 130 ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_two"); 131 mmap_ptr = mmap(NULL, 2 * page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size); 132 ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_all"); 133 134 /* good read-only pages */ 135 mmap_ptr = mmap(NULL, 4 * page_size, PROT_READ, MAP_SHARED, rb_fd, 0); 136 if (!ASSERT_OK_PTR(mmap_ptr, "ro_prod_pos")) 137 goto cleanup; 138 139 ASSERT_ERR(mprotect(mmap_ptr, 4 * page_size, PROT_WRITE), "write_protect"); 140 ASSERT_ERR(mprotect(mmap_ptr, 4 * page_size, PROT_EXEC), "exec_protect"); 141 ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 4 * page_size, MREMAP_MAYMOVE), "ro_remap"); 142 ASSERT_OK(munmap(mmap_ptr, 4 * page_size), "unmap_ro"); 143 144 /* good read-only pages with initial offset */ 145 mmap_ptr = mmap(NULL, page_size, PROT_READ, MAP_SHARED, rb_fd, page_size); 146 if (!ASSERT_OK_PTR(mmap_ptr, "ro_prod_pos")) 147 goto cleanup; 148 149 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_WRITE), "write_protect"); 150 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_protect"); 151 ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 3 * page_size, MREMAP_MAYMOVE), "ro_remap"); 152 ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_ro"); 153 154 /* only trigger BPF program for current process */ 155 skel->bss->pid = getpid(); 156 157 ringbuf = ring_buffer__new(skel->maps.ringbuf.map_fd, 158 process_sample, NULL, NULL); 159 if (CHECK(!ringbuf, "ringbuf_create", "failed to create ringbuf\n")) 160 goto cleanup; 161 162 err = test_ringbuf_lskel__attach(skel); 163 if (CHECK(err, "skel_attach", "skeleton attachment failed: %d\n", err)) 164 goto cleanup; 165 166 trigger_samples(); 167 168 ring = ring_buffer__ring(ringbuf, 0); 169 if (!ASSERT_OK_PTR(ring, "ring_buffer__ring_idx_0")) 170 goto cleanup; 171 172 map_fd = ring__map_fd(ring); 173 ASSERT_EQ(map_fd, skel->maps.ringbuf.map_fd, "ring_map_fd"); 174 175 /* 2 submitted + 1 discarded records */ 176 CHECK(skel->bss->avail_data != 3 * rec_sz, 177 "err_avail_size", "exp %ld, got %ld\n", 178 3L * rec_sz, skel->bss->avail_data); 179 CHECK(skel->bss->ring_size != page_size, 180 "err_ring_size", "exp %ld, got %ld\n", 181 (long)page_size, skel->bss->ring_size); 182 CHECK(skel->bss->cons_pos != 0, 183 "err_cons_pos", "exp %ld, got %ld\n", 184 0L, skel->bss->cons_pos); 185 CHECK(skel->bss->prod_pos != 3 * rec_sz, 186 "err_prod_pos", "exp %ld, got %ld\n", 187 3L * rec_sz, skel->bss->prod_pos); 188 189 /* verify getting this data directly via the ring object yields the same 190 * results 191 */ 192 avail_data = ring__avail_data_size(ring); 193 ASSERT_EQ(avail_data, 3 * rec_sz, "ring_avail_size"); 194 ring_size = ring__size(ring); 195 ASSERT_EQ(ring_size, page_size, "ring_ring_size"); 196 cons_pos = ring__consumer_pos(ring); 197 ASSERT_EQ(cons_pos, 0, "ring_cons_pos"); 198 prod_pos = ring__producer_pos(ring); 199 ASSERT_EQ(prod_pos, 3 * rec_sz, "ring_prod_pos"); 200 201 /* poll for samples */ 202 err = ring_buffer__poll(ringbuf, -1); 203 204 /* -EDONE is used as an indicator that we are done */ 205 if (CHECK(err != -EDONE, "err_done", "done err: %d\n", err)) 206 goto cleanup; 207 cnt = atomic_xchg(&sample_cnt, 0); 208 CHECK(cnt != 2, "cnt", "exp %d samples, got %d\n", 2, cnt); 209 210 /* we expect extra polling to return nothing */ 211 err = ring_buffer__poll(ringbuf, 0); 212 if (CHECK(err != 0, "extra_samples", "poll result: %d\n", err)) 213 goto cleanup; 214 cnt = atomic_xchg(&sample_cnt, 0); 215 CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt); 216 217 CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n", 218 0L, skel->bss->dropped); 219 CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n", 220 2L, skel->bss->total); 221 CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n", 222 1L, skel->bss->discarded); 223 224 /* now validate consumer position is updated and returned */ 225 trigger_samples(); 226 CHECK(skel->bss->cons_pos != 3 * rec_sz, 227 "err_cons_pos", "exp %ld, got %ld\n", 228 3L * rec_sz, skel->bss->cons_pos); 229 err = ring_buffer__poll(ringbuf, -1); 230 CHECK(err <= 0, "poll_err", "err %d\n", err); 231 cnt = atomic_xchg(&sample_cnt, 0); 232 CHECK(cnt != 2, "cnt", "exp %d samples, got %d\n", 2, cnt); 233 234 /* start poll in background w/ long timeout */ 235 err = pthread_create(&thread, NULL, poll_thread, (void *)(long)10000); 236 if (CHECK(err, "bg_poll", "pthread_create failed: %d\n", err)) 237 goto cleanup; 238 239 /* turn off notifications now */ 240 skel->bss->flags = BPF_RB_NO_WAKEUP; 241 242 /* give background thread a bit of a time */ 243 usleep(50000); 244 trigger_samples(); 245 /* sleeping arbitrarily is bad, but no better way to know that 246 * epoll_wait() **DID NOT** unblock in background thread 247 */ 248 usleep(50000); 249 /* background poll should still be blocked */ 250 err = pthread_tryjoin_np(thread, (void **)&bg_ret); 251 if (CHECK(err != EBUSY, "try_join", "err %d\n", err)) 252 goto cleanup; 253 254 /* BPF side did everything right */ 255 CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n", 256 0L, skel->bss->dropped); 257 CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n", 258 2L, skel->bss->total); 259 CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n", 260 1L, skel->bss->discarded); 261 cnt = atomic_xchg(&sample_cnt, 0); 262 CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt); 263 264 /* clear flags to return to "adaptive" notification mode */ 265 skel->bss->flags = 0; 266 267 /* produce new samples, no notification should be triggered, because 268 * consumer is now behind 269 */ 270 trigger_samples(); 271 272 /* background poll should still be blocked */ 273 err = pthread_tryjoin_np(thread, (void **)&bg_ret); 274 if (CHECK(err != EBUSY, "try_join", "err %d\n", err)) 275 goto cleanup; 276 277 /* still no samples, because consumer is behind */ 278 cnt = atomic_xchg(&sample_cnt, 0); 279 CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt); 280 281 skel->bss->dropped = 0; 282 skel->bss->total = 0; 283 skel->bss->discarded = 0; 284 285 skel->bss->value = 333; 286 syscall(__NR_getpgid); 287 /* now force notifications */ 288 skel->bss->flags = BPF_RB_FORCE_WAKEUP; 289 skel->bss->value = 777; 290 syscall(__NR_getpgid); 291 292 /* now we should get a pending notification */ 293 usleep(50000); 294 err = pthread_tryjoin_np(thread, (void **)&bg_ret); 295 if (CHECK(err, "join_bg", "err %d\n", err)) 296 goto cleanup; 297 298 if (CHECK(bg_ret <= 0, "bg_ret", "epoll_wait result: %ld", bg_ret)) 299 goto cleanup; 300 301 /* due to timing variations, there could still be non-notified 302 * samples, so consume them here to collect all the samples 303 */ 304 err = ring_buffer__consume(ringbuf); 305 CHECK(err < 0, "rb_consume", "failed: %d\b", err); 306 307 /* also consume using ring__consume to make sure it works the same */ 308 err = ring__consume(ring); 309 ASSERT_GE(err, 0, "ring_consume"); 310 311 /* 3 rounds, 2 samples each */ 312 cnt = atomic_xchg(&sample_cnt, 0); 313 CHECK(cnt != 6, "cnt", "exp %d samples, got %d\n", 6, cnt); 314 315 /* BPF side did everything right */ 316 CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n", 317 0L, skel->bss->dropped); 318 CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n", 319 2L, skel->bss->total); 320 CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n", 321 1L, skel->bss->discarded); 322 323 test_ringbuf_lskel__detach(skel); 324 cleanup: 325 ring_buffer__free(ringbuf); 326 test_ringbuf_lskel__destroy(skel); 327 } 328 329 static int process_map_key_sample(void *ctx, void *data, size_t len) 330 { 331 struct sample *s; 332 int err, val; 333 334 s = data; 335 switch (s->seq) { 336 case 1: 337 ASSERT_EQ(s->value, 42, "sample_value"); 338 err = bpf_map_lookup_elem(skel_map_key->maps.hash_map.map_fd, 339 s, &val); 340 ASSERT_OK(err, "hash_map bpf_map_lookup_elem"); 341 ASSERT_EQ(val, 1, "hash_map val"); 342 return -EDONE; 343 default: 344 return 0; 345 } 346 } 347 348 static void ringbuf_map_key_subtest(void) 349 { 350 int err; 351 352 skel_map_key = test_ringbuf_map_key_lskel__open(); 353 if (!ASSERT_OK_PTR(skel_map_key, "test_ringbuf_map_key_lskel__open")) 354 return; 355 356 skel_map_key->maps.ringbuf.max_entries = getpagesize(); 357 skel_map_key->bss->pid = getpid(); 358 359 err = test_ringbuf_map_key_lskel__load(skel_map_key); 360 if (!ASSERT_OK(err, "test_ringbuf_map_key_lskel__load")) 361 goto cleanup; 362 363 ringbuf = ring_buffer__new(skel_map_key->maps.ringbuf.map_fd, 364 process_map_key_sample, NULL, NULL); 365 if (!ASSERT_OK_PTR(ringbuf, "ring_buffer__new")) 366 goto cleanup; 367 368 err = test_ringbuf_map_key_lskel__attach(skel_map_key); 369 if (!ASSERT_OK(err, "test_ringbuf_map_key_lskel__attach")) 370 goto cleanup_ringbuf; 371 372 syscall(__NR_getpgid); 373 ASSERT_EQ(skel_map_key->bss->seq, 1, "skel_map_key->bss->seq"); 374 err = ring_buffer__poll(ringbuf, -1); 375 ASSERT_EQ(err, -EDONE, "ring_buffer__poll"); 376 377 cleanup_ringbuf: 378 ring_buffer__free(ringbuf); 379 cleanup: 380 test_ringbuf_map_key_lskel__destroy(skel_map_key); 381 } 382 383 void test_ringbuf(void) 384 { 385 if (test__start_subtest("ringbuf")) 386 ringbuf_subtest(); 387 if (test__start_subtest("ringbuf_map_key")) 388 ringbuf_map_key_subtest(); 389 } 390