1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Stress every LRU lock-failure and orphan-recovery. 4 * perf_event NMI BPF on every online CPU does 5 * update+delete on a small LRU map; userspace threads on every CPU do 6 * the same from syscall context. 7 */ 8 #define _GNU_SOURCE 9 #include <pthread.h> 10 #include <sched.h> 11 #include <sys/syscall.h> 12 #include <linux/perf_event.h> 13 #include <test_progs.h> 14 #include "testing_helpers.h" 15 #include "lru_lock_nmi.skel.h" 16 17 #define MAP_ENTRIES 64 18 #define KEY_RANGE (MAP_ENTRIES * 2) 19 #define STRESS_NS (500 * 1000 * 1000ULL) 20 21 struct hammer_arg { 22 int map_fd; 23 int cpu; 24 __u64 deadline_ns; 25 }; 26 27 struct refill_arg { 28 int map_fd; 29 int cpu; 30 int per_cpu_quota; 31 int update_errors; 32 }; 33 34 /* 35 * Pin the calling thread to @cpu. Uses dynamically-allocated CPU sets so 36 * we stay correct on hosts with @cpu >= CPU_SETSIZE (default 1024). 37 */ 38 static int pin_to_cpu(int cpu) 39 { 40 cpu_set_t *cs; 41 size_t cs_size; 42 int err; 43 44 cs = CPU_ALLOC(cpu + 1); 45 if (!cs) 46 return -ENOMEM; 47 cs_size = CPU_ALLOC_SIZE(cpu + 1); 48 49 CPU_ZERO_S(cs_size, cs); 50 CPU_SET_S(cpu, cs_size, cs); 51 err = pthread_setaffinity_np(pthread_self(), cs_size, cs); 52 CPU_FREE(cs); 53 return err; 54 } 55 56 static void *hammer_thread(void *p) 57 { 58 struct hammer_arg *a = p; 59 int nr_possible_cpus = libbpf_num_possible_cpus(); 60 __u64 val[nr_possible_cpus]; 61 unsigned int seed; 62 __u32 key; 63 64 memset(val, 0, sizeof(val)); 65 pin_to_cpu(a->cpu); 66 67 seed = (unsigned int)a->cpu ^ (unsigned int)(uintptr_t)pthread_self(); 68 69 while (get_time_ns() < a->deadline_ns) { 70 bool do_update = rand_r(&seed) & 1; 71 72 key = rand_r(&seed) % KEY_RANGE; 73 if (do_update) 74 bpf_map_update_elem(a->map_fd, &key, val, BPF_ANY); 75 else 76 bpf_map_delete_elem(a->map_fd, &key); 77 } 78 return NULL; 79 } 80 81 static void *refill_thread(void *p) 82 { 83 struct refill_arg *a = p; 84 int nr_possible_cpus = libbpf_num_possible_cpus(); 85 __u64 val[nr_possible_cpus]; 86 __u32 start, end, key; 87 88 memset(val, 0, sizeof(val)); 89 pin_to_cpu(a->cpu); 90 91 start = (__u32)a->cpu * (__u32)a->per_cpu_quota; 92 end = start + (__u32)a->per_cpu_quota; 93 for (key = start; key < end; key++) 94 if (bpf_map_update_elem(a->map_fd, &key, val, BPF_ANY)) 95 a->update_errors++; 96 return NULL; 97 } 98 99 /* 100 * Drain the map, then refill it with each CPU inserting only its own 101 * quota of keys. 102 * After refill, lookup every key we inserted - a stranded node on any 103 * CPU's pool would have forced eviction. 104 */ 105 static int drain_then_verify_capacity(int map_fd, int nr_cpus) 106 { 107 int per_cpu_quota = MAP_ENTRIES / nr_cpus; 108 int total = per_cpu_quota * nr_cpus; 109 int nr_possible_cpus = libbpf_num_possible_cpus(); 110 pthread_t threads[nr_cpus]; 111 struct refill_arg args[nr_cpus]; 112 __u64 val[nr_possible_cpus]; 113 int i, hits = 0, nthreads = 0; 114 __u32 key; 115 116 memset(val, 0, sizeof(val)); 117 118 for (key = 0; key < KEY_RANGE; key++) 119 bpf_map_delete_elem(map_fd, &key); 120 121 for (i = 0; i < nr_cpus; i++) { 122 args[i] = (struct refill_arg){ 123 .map_fd = map_fd, 124 .cpu = i, 125 .per_cpu_quota = per_cpu_quota, 126 }; 127 if (pthread_create(&threads[nthreads], NULL, refill_thread, &args[i]) == 0) 128 nthreads++; 129 } 130 for (i = 0; i < nthreads; i++) 131 pthread_join(threads[i], NULL); 132 133 for (i = 0; i < nr_cpus; i++) 134 if (args[i].update_errors) 135 return -ENOMEM; 136 137 for (key = 0; key < (__u32)total; key++) 138 if (bpf_map_lookup_elem(map_fd, &key, val) == 0) 139 hits++; 140 141 return hits == total ? 0 : -EIO; 142 } 143 144 static void run_variant(enum bpf_map_type type, __u32 map_flags, const char *name) 145 { 146 struct perf_event_attr attr = { 147 .size = sizeof(attr), 148 .type = PERF_TYPE_HARDWARE, 149 .config = PERF_COUNT_HW_CPU_CYCLES, 150 .freq = 1, 151 }; 152 int nr_cpus, max_cpus = 64; 153 struct bpf_link *links[max_cpus]; 154 pthread_t threads[max_cpus]; 155 struct hammer_arg args[max_cpus]; 156 struct lru_lock_nmi *skel = NULL; 157 int map_fd, i, err, nr_threads = 0, pmu_fd = -1; 158 __u64 deadline; 159 160 nr_cpus = libbpf_num_possible_cpus(); 161 if (!ASSERT_GT(nr_cpus, 0, "num_cpus")) 162 return; 163 164 if (nr_cpus > max_cpus) 165 nr_cpus = max_cpus; 166 167 if (!test__start_subtest(name)) 168 return; 169 170 memset(links, 0, sizeof(links)); 171 skel = lru_lock_nmi__open(); 172 if (!ASSERT_OK_PTR(skel, "skel_open")) 173 goto cleanup; 174 175 err = bpf_map__set_type(skel->maps.lru_map, type); 176 if (!ASSERT_OK(err, "set_type")) 177 goto cleanup; 178 err = bpf_map__set_map_flags(skel->maps.lru_map, map_flags); 179 if (!ASSERT_OK(err, "set_flags")) 180 goto cleanup; 181 err = bpf_map__set_max_entries(skel->maps.lru_map, MAP_ENTRIES); 182 if (!ASSERT_OK(err, "set_max_entries")) 183 goto cleanup; 184 185 err = lru_lock_nmi__load(skel); 186 if (!ASSERT_OK(err, "skel_load")) 187 goto cleanup; 188 189 skel->bss->hits = 0; 190 map_fd = bpf_map__fd(skel->maps.lru_map); 191 attr.sample_freq = read_perf_max_sample_freq(); 192 193 for (i = 0; i < nr_cpus; i++) { 194 pmu_fd = syscall(__NR_perf_event_open, &attr, -1, i, -1, 0); 195 if (pmu_fd < 0) { 196 if (i == 0 && 197 (errno == ENOENT || errno == EOPNOTSUPP)) { 198 test__skip(); 199 goto cleanup; 200 } 201 continue; 202 } 203 /* libbpf takes ownership of pfd on success */ 204 links[i] = bpf_program__attach_perf_event(skel->progs.oncpu, pmu_fd); 205 if (!links[i]) 206 close(pmu_fd); 207 } 208 209 deadline = get_time_ns() + STRESS_NS; 210 for (i = 0; i < nr_cpus; i++) { 211 args[i].map_fd = map_fd; 212 args[i].cpu = i; 213 args[i].deadline_ns = deadline; 214 if (pthread_create(&threads[nr_threads], NULL, hammer_thread, &args[i]) == 0) 215 nr_threads++; 216 } 217 for (i = 0; i < nr_threads; i++) 218 pthread_join(threads[i], NULL); 219 220 for (i = 0; i < nr_cpus; i++) { 221 if (links[i]) { 222 bpf_link__destroy(links[i]); 223 links[i] = NULL; 224 } 225 } 226 227 ASSERT_GT(skel->bss->hits, 0, "nmi_bpf_ran"); 228 ASSERT_OK(drain_then_verify_capacity(map_fd, nr_cpus), "drain_then_verify_capacity"); 229 230 cleanup: 231 for (i = 0; i < nr_cpus; i++) { 232 if (links[i]) 233 bpf_link__destroy(links[i]); 234 } 235 lru_lock_nmi__destroy(skel); 236 } 237 238 void serial_test_lru_lock_nmi(void) 239 { 240 run_variant(BPF_MAP_TYPE_LRU_HASH, 0, "common_lru"); 241 run_variant(BPF_MAP_TYPE_LRU_HASH, BPF_F_NO_COMMON_LRU, "no_common_lru"); 242 run_variant(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0, "percpu_lru"); 243 } 244