1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2023. Huawei Technologies Co., Ltd */ 3 #include <argp.h> 4 #include <stdbool.h> 5 #include <pthread.h> 6 #include <sys/types.h> 7 #include <sys/stat.h> 8 #include <sys/param.h> 9 #include <fcntl.h> 10 11 #include "bench.h" 12 #include "bpf_util.h" 13 #include "cgroup_helpers.h" 14 #include "htab_mem_bench.skel.h" 15 16 struct htab_mem_use_case { 17 const char *name; 18 const char **progs; 19 /* Do synchronization between addition thread and deletion thread */ 20 bool need_sync; 21 }; 22 23 static struct htab_mem_ctx { 24 const struct htab_mem_use_case *uc; 25 struct htab_mem_bench *skel; 26 pthread_barrier_t *notify; 27 int fd; 28 } ctx; 29 30 const char *ow_progs[] = {"overwrite", NULL}; 31 const char *batch_progs[] = {"batch_add_batch_del", NULL}; 32 const char *add_del_progs[] = {"add_only", "del_only", NULL}; 33 const static struct htab_mem_use_case use_cases[] = { 34 { .name = "overwrite", .progs = ow_progs }, 35 { .name = "batch_add_batch_del", .progs = batch_progs }, 36 { .name = "add_del_on_diff_cpu", .progs = add_del_progs, .need_sync = true }, 37 }; 38 39 static struct htab_mem_args { 40 u32 value_size; 41 const char *use_case; 42 bool preallocated; 43 } args = { 44 .value_size = 8, 45 .use_case = "overwrite", 46 .preallocated = false, 47 }; 48 49 enum { 50 ARG_VALUE_SIZE = 10000, 51 ARG_USE_CASE = 10001, 52 ARG_PREALLOCATED = 10002, 53 }; 54 55 static const struct argp_option opts[] = { 56 { "value-size", ARG_VALUE_SIZE, "VALUE_SIZE", 0, 57 "Set the value size of hash map (default 8)" }, 58 { "use-case", ARG_USE_CASE, "USE_CASE", 0, 59 "Set the use case of hash map: overwrite|batch_add_batch_del|add_del_on_diff_cpu" }, 60 { "preallocated", ARG_PREALLOCATED, NULL, 0, "use preallocated hash map" }, 61 {}, 62 }; 63 64 static error_t htab_mem_parse_arg(int key, char *arg, struct argp_state *state) 65 { 66 switch (key) { 67 case ARG_VALUE_SIZE: 68 args.value_size = strtoul(arg, NULL, 10); 69 if (args.value_size > 4096) { 70 fprintf(stderr, "too big value size %u\n", args.value_size); 71 argp_usage(state); 72 } 73 break; 74 case ARG_USE_CASE: 75 args.use_case = strdup(arg); 76 if (!args.use_case) { 77 fprintf(stderr, "no mem for use-case\n"); 78 argp_usage(state); 79 } 80 break; 81 case ARG_PREALLOCATED: 82 args.preallocated = true; 83 break; 84 default: 85 return ARGP_ERR_UNKNOWN; 86 } 87 88 return 0; 89 } 90 91 const struct argp bench_htab_mem_argp = { 92 .options = opts, 93 .parser = htab_mem_parse_arg, 94 }; 95 96 static void htab_mem_validate(void) 97 { 98 if (!strcmp(use_cases[2].name, args.use_case) && env.producer_cnt % 2) { 99 fprintf(stderr, "%s needs an even number of producers\n", args.use_case); 100 exit(1); 101 } 102 } 103 104 static int htab_mem_bench_init_barriers(void) 105 { 106 pthread_barrier_t *barriers; 107 unsigned int i, nr; 108 109 if (!ctx.uc->need_sync) 110 return 0; 111 112 nr = (env.producer_cnt + 1) / 2; 113 barriers = calloc(nr, sizeof(*barriers)); 114 if (!barriers) 115 return -1; 116 117 /* Used for synchronization between two threads */ 118 for (i = 0; i < nr; i++) 119 pthread_barrier_init(&barriers[i], NULL, 2); 120 121 ctx.notify = barriers; 122 return 0; 123 } 124 125 static void htab_mem_bench_exit_barriers(void) 126 { 127 unsigned int i, nr; 128 129 if (!ctx.notify) 130 return; 131 132 nr = (env.producer_cnt + 1) / 2; 133 for (i = 0; i < nr; i++) 134 pthread_barrier_destroy(&ctx.notify[i]); 135 free(ctx.notify); 136 } 137 138 static const struct htab_mem_use_case *htab_mem_find_use_case_or_exit(const char *name) 139 { 140 unsigned int i; 141 142 for (i = 0; i < ARRAY_SIZE(use_cases); i++) { 143 if (!strcmp(name, use_cases[i].name)) 144 return &use_cases[i]; 145 } 146 147 fprintf(stderr, "no such use-case: %s\n", name); 148 fprintf(stderr, "available use case:"); 149 for (i = 0; i < ARRAY_SIZE(use_cases); i++) 150 fprintf(stderr, " %s", use_cases[i].name); 151 fprintf(stderr, "\n"); 152 exit(1); 153 } 154 155 static void htab_mem_setup_impl(enum bpf_map_type map_type) 156 { 157 struct bpf_map *map; 158 const char **names; 159 int err; 160 161 setup_libbpf(); 162 163 ctx.uc = htab_mem_find_use_case_or_exit(args.use_case); 164 err = htab_mem_bench_init_barriers(); 165 if (err) { 166 fprintf(stderr, "failed to init barrier\n"); 167 exit(1); 168 } 169 170 ctx.fd = cgroup_setup_and_join("/htab_mem"); 171 if (ctx.fd < 0) 172 goto cleanup; 173 174 ctx.skel = htab_mem_bench__open(); 175 if (!ctx.skel) { 176 fprintf(stderr, "failed to open skeleton\n"); 177 goto cleanup; 178 } 179 180 map = ctx.skel->maps.htab; 181 bpf_map__set_type(map, map_type); 182 bpf_map__set_value_size(map, args.value_size); 183 /* Ensure that different CPUs can operate on different subset */ 184 bpf_map__set_max_entries(map, MAX(8192, 64 * env.nr_cpus)); 185 if (map_type != BPF_MAP_TYPE_RHASH && args.preallocated) 186 bpf_map__set_map_flags(map, bpf_map__map_flags(map) & ~BPF_F_NO_PREALLOC); 187 188 names = ctx.uc->progs; 189 while (*names) { 190 struct bpf_program *prog; 191 192 prog = bpf_object__find_program_by_name(ctx.skel->obj, *names); 193 if (!prog) { 194 fprintf(stderr, "no such program %s\n", *names); 195 goto cleanup; 196 } 197 bpf_program__set_autoload(prog, true); 198 names++; 199 } 200 ctx.skel->bss->nr_thread = env.producer_cnt; 201 202 err = htab_mem_bench__load(ctx.skel); 203 if (err) { 204 fprintf(stderr, "failed to load skeleton\n"); 205 goto cleanup; 206 } 207 err = htab_mem_bench__attach(ctx.skel); 208 if (err) { 209 fprintf(stderr, "failed to attach skeleton\n"); 210 goto cleanup; 211 } 212 return; 213 214 cleanup: 215 htab_mem_bench__destroy(ctx.skel); 216 htab_mem_bench_exit_barriers(); 217 if (ctx.fd >= 0) { 218 close(ctx.fd); 219 cleanup_cgroup_environment(); 220 } 221 exit(1); 222 } 223 224 static void htab_mem_setup(void) 225 { 226 htab_mem_setup_impl(BPF_MAP_TYPE_HASH); 227 } 228 229 static void rhtab_mem_setup(void) 230 { 231 htab_mem_setup_impl(BPF_MAP_TYPE_RHASH); 232 } 233 234 static void htab_mem_add_fn(pthread_barrier_t *notify) 235 { 236 while (true) { 237 /* Do addition */ 238 (void)syscall(__NR_getpgid, 0); 239 /* Notify deletion thread to do deletion */ 240 pthread_barrier_wait(notify); 241 /* Wait for deletion to complete */ 242 pthread_barrier_wait(notify); 243 } 244 } 245 246 static void htab_mem_delete_fn(pthread_barrier_t *notify) 247 { 248 while (true) { 249 /* Wait for addition to complete */ 250 pthread_barrier_wait(notify); 251 /* Do deletion */ 252 (void)syscall(__NR_getppid); 253 /* Notify addition thread to do addition */ 254 pthread_barrier_wait(notify); 255 } 256 } 257 258 static void *htab_mem_producer(void *arg) 259 { 260 pthread_barrier_t *notify; 261 int seq; 262 263 if (!ctx.uc->need_sync) { 264 while (true) 265 (void)syscall(__NR_getpgid, 0); 266 return NULL; 267 } 268 269 seq = (long)arg; 270 notify = &ctx.notify[seq / 2]; 271 if (seq & 1) 272 htab_mem_delete_fn(notify); 273 else 274 htab_mem_add_fn(notify); 275 return NULL; 276 } 277 278 static void htab_mem_read_mem_cgrp_file(const char *name, unsigned long *value) 279 { 280 char buf[32]; 281 ssize_t got; 282 int fd; 283 284 fd = openat(ctx.fd, name, O_RDONLY); 285 if (fd < 0) { 286 /* cgroup v1 ? */ 287 fprintf(stderr, "no %s\n", name); 288 *value = 0; 289 return; 290 } 291 292 got = read(fd, buf, sizeof(buf) - 1); 293 close(fd); 294 if (got <= 0) { 295 *value = 0; 296 return; 297 } 298 buf[got] = 0; 299 300 *value = strtoull(buf, NULL, 0); 301 } 302 303 static void htab_mem_measure(struct bench_res *res) 304 { 305 res->hits = atomic_swap(&ctx.skel->bss->op_cnt, 0) / env.producer_cnt; 306 htab_mem_read_mem_cgrp_file("memory.current", &res->gp_ct); 307 } 308 309 static void htab_mem_report_progress(int iter, struct bench_res *res, long delta_ns) 310 { 311 double loop, mem; 312 313 loop = res->hits / 1000.0 / (delta_ns / 1000000000.0); 314 mem = res->gp_ct / 1048576.0; 315 printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0); 316 printf("per-prod-op %7.2lfk/s, memory usage %7.2lfMiB\n", loop, mem); 317 } 318 319 static void htab_mem_report_final(struct bench_res res[], int res_cnt) 320 { 321 double mem_mean = 0.0, mem_stddev = 0.0; 322 double loop_mean = 0.0, loop_stddev = 0.0; 323 unsigned long peak_mem; 324 int i; 325 326 for (i = 0; i < res_cnt; i++) { 327 loop_mean += res[i].hits / 1000.0 / (0.0 + res_cnt); 328 mem_mean += res[i].gp_ct / 1048576.0 / (0.0 + res_cnt); 329 } 330 if (res_cnt > 1) { 331 for (i = 0; i < res_cnt; i++) { 332 loop_stddev += (loop_mean - res[i].hits / 1000.0) * 333 (loop_mean - res[i].hits / 1000.0) / 334 (res_cnt - 1.0); 335 mem_stddev += (mem_mean - res[i].gp_ct / 1048576.0) * 336 (mem_mean - res[i].gp_ct / 1048576.0) / 337 (res_cnt - 1.0); 338 } 339 loop_stddev = sqrt(loop_stddev); 340 mem_stddev = sqrt(mem_stddev); 341 } 342 343 htab_mem_read_mem_cgrp_file("memory.peak", &peak_mem); 344 printf("Summary: per-prod-op %7.2lf \u00B1 %7.2lfk/s, memory usage %7.2lf \u00B1 %7.2lfMiB," 345 " peak memory usage %7.2lfMiB\n", 346 loop_mean, loop_stddev, mem_mean, mem_stddev, peak_mem / 1048576.0); 347 348 close(ctx.fd); 349 cleanup_cgroup_environment(); 350 } 351 352 static void rhtab_mem_validate(void) 353 { 354 if (args.preallocated) { 355 fprintf(stderr, "rhash map does not support preallocation\n"); 356 exit(1); 357 } 358 htab_mem_validate(); 359 } 360 361 const struct bench bench_htab_mem = { 362 .name = "htab-mem", 363 .argp = &bench_htab_mem_argp, 364 .validate = htab_mem_validate, 365 .setup = htab_mem_setup, 366 .producer_thread = htab_mem_producer, 367 .measure = htab_mem_measure, 368 .report_progress = htab_mem_report_progress, 369 .report_final = htab_mem_report_final, 370 }; 371 372 const struct bench bench_rhtab_mem = { 373 .name = "rhtab-mem", 374 .argp = &bench_htab_mem_argp, 375 .validate = rhtab_mem_validate, 376 .setup = rhtab_mem_setup, 377 .producer_thread = htab_mem_producer, 378 .measure = htab_mem_measure, 379 .report_progress = htab_mem_report_progress, 380 .report_final = htab_mem_report_final, 381 }; 382