1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2011-2017, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com> 4 * 5 * Parts came from evlist.c builtin-{top,stat,record}.c, see those files for further 6 * copyright notes. 7 */ 8 9 #include <sys/mman.h> 10 #include <errno.h> 11 #include <inttypes.h> 12 #include <asm/bug.h> 13 #include <linux/zalloc.h> 14 #include <stdlib.h> 15 #include <string.h> 16 #include <unistd.h> // sysconf() 17 #include <perf/mmap.h> 18 #ifdef HAVE_LIBNUMA_SUPPORT 19 #include <numaif.h> 20 #endif 21 #include "cpumap.h" 22 #include "debug.h" 23 #include "event.h" 24 #include "mmap.h" 25 #include "../perf.h" 26 #include <internal/lib.h> /* page_size */ 27 #include <linux/bitmap.h> 28 29 #define MASK_SIZE 1023 30 void mmap_cpu_mask__scnprintf(struct mmap_cpu_mask *mask, const char *tag) 31 { 32 char buf[MASK_SIZE + 1]; 33 size_t len; 34 35 len = bitmap_scnprintf(mask->bits, mask->nbits, buf, MASK_SIZE); 36 buf[len] = '\0'; 37 pr_debug("%p: %s mask[%zd]: %s\n", mask, tag, mask->nbits, buf); 38 } 39 40 size_t mmap__mmap_len(struct mmap *map) 41 { 42 return perf_mmap__mmap_len(&map->core); 43 } 44 45 int __weak auxtrace_mmap__mmap(struct auxtrace_mmap *mm __maybe_unused, 46 struct auxtrace_mmap_params *mp __maybe_unused, 47 void *userpg __maybe_unused, 48 int fd __maybe_unused) 49 { 50 return 0; 51 } 52 53 void __weak auxtrace_mmap__munmap(struct auxtrace_mmap *mm __maybe_unused) 54 { 55 } 56 57 void __weak auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp __maybe_unused, 58 off_t auxtrace_offset __maybe_unused, 59 unsigned int auxtrace_pages __maybe_unused, 60 bool auxtrace_overwrite __maybe_unused) 61 { 62 } 63 64 void __weak auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp __maybe_unused, 65 struct evlist *evlist __maybe_unused, 66 struct evsel *evsel __maybe_unused, 67 int idx __maybe_unused) 68 { 69 } 70 71 #ifdef HAVE_AIO_SUPPORT 72 static int perf_mmap__aio_enabled(struct mmap *map) 73 { 74 return map->aio.nr_cblocks > 0; 75 } 76 77 #ifdef HAVE_LIBNUMA_SUPPORT 78 static int perf_mmap__aio_alloc(struct mmap *map, int idx) 79 { 80 map->aio.data[idx] = mmap(NULL, mmap__mmap_len(map), PROT_READ|PROT_WRITE, 81 MAP_PRIVATE|MAP_ANONYMOUS, 0, 0); 82 if (map->aio.data[idx] == MAP_FAILED) { 83 map->aio.data[idx] = NULL; 84 return -1; 85 } 86 87 return 0; 88 } 89 90 static void perf_mmap__aio_free(struct mmap *map, int idx) 91 { 92 if (!map->aio.data || !map->aio.data[idx]) 93 return; 94 munmap(map->aio.data[idx], mmap__mmap_len(map)); 95 map->aio.data[idx] = NULL; 96 } 97 98 static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, int affinity) 99 { 100 void *data; 101 size_t mmap_len; 102 unsigned long *node_mask; 103 unsigned long node_index; 104 int err = 0; 105 106 if (affinity != PERF_AFFINITY_SYS && cpu__max_node() > 1) { 107 int node; 108 109 data = map->aio.data[idx]; 110 mmap_len = mmap__mmap_len(map); 111 node = cpu__get_node(cpu); 112 /* -1 sign-extends to ULONG_MAX, wrapping bitmap_zalloc(0) and OOB __set_bit */ 113 if (node < 0) 114 return 0; 115 node_index = node; 116 node_mask = bitmap_zalloc(node_index + 1); 117 if (!node_mask) { 118 pr_err("Failed to allocate node mask for mbind: error %m\n"); 119 return -1; 120 } 121 __set_bit(node_index, node_mask); 122 if (mbind(data, mmap_len, MPOL_BIND, node_mask, node_index + 1 + 1, 0)) { 123 pr_err("Failed to bind [%p-%p] AIO buffer to node %lu: error %m\n", 124 data, data + mmap_len, node_index); 125 err = -1; 126 } 127 bitmap_free(node_mask); 128 } 129 130 return err; 131 } 132 #else /* !HAVE_LIBNUMA_SUPPORT */ 133 static int perf_mmap__aio_alloc(struct mmap *map, int idx) 134 { 135 map->aio.data[idx] = malloc(mmap__mmap_len(map)); 136 if (map->aio.data[idx] == NULL) 137 return -1; 138 139 return 0; 140 } 141 142 static void perf_mmap__aio_free(struct mmap *map, int idx) 143 { 144 if (!map->aio.data) 145 return; 146 zfree(&(map->aio.data[idx])); 147 } 148 149 static int perf_mmap__aio_bind(struct mmap *map __maybe_unused, int idx __maybe_unused, 150 struct perf_cpu cpu __maybe_unused, int affinity __maybe_unused) 151 { 152 return 0; 153 } 154 #endif 155 156 static int perf_mmap__aio_mmap(struct mmap *map, struct mmap_params *mp) 157 { 158 int delta_max, i, prio, ret; 159 160 map->aio.nr_cblocks = mp->nr_cblocks; 161 if (map->aio.nr_cblocks) { 162 map->aio.aiocb = calloc(map->aio.nr_cblocks, sizeof(struct aiocb *)); 163 if (!map->aio.aiocb) { 164 pr_debug2("failed to allocate aiocb for data buffer, error %m\n"); 165 return -1; 166 } 167 map->aio.cblocks = calloc(map->aio.nr_cblocks, sizeof(struct aiocb)); 168 if (!map->aio.cblocks) { 169 pr_debug2("failed to allocate cblocks for data buffer, error %m\n"); 170 return -1; 171 } 172 map->aio.data = calloc(map->aio.nr_cblocks, sizeof(void *)); 173 if (!map->aio.data) { 174 pr_debug2("failed to allocate data buffer, error %m\n"); 175 return -1; 176 } 177 delta_max = sysconf(_SC_AIO_PRIO_DELTA_MAX); 178 for (i = 0; i < map->aio.nr_cblocks; ++i) { 179 ret = perf_mmap__aio_alloc(map, i); 180 if (ret == -1) { 181 pr_debug2("failed to allocate data buffer area, error %m"); 182 return -1; 183 } 184 ret = perf_mmap__aio_bind(map, i, map->core.cpu, mp->affinity); 185 if (ret == -1) 186 return -1; 187 /* 188 * Use cblock.aio_fildes value different from -1 189 * to denote started aio write operation on the 190 * cblock so it requires explicit record__aio_sync() 191 * call prior the cblock may be reused again. 192 */ 193 map->aio.cblocks[i].aio_fildes = -1; 194 /* 195 * Allocate cblocks with priority delta to have 196 * faster aio write system calls because queued requests 197 * are kept in separate per-prio queues and adding 198 * a new request will iterate thru shorter per-prio 199 * list. Blocks with numbers higher than 200 * _SC_AIO_PRIO_DELTA_MAX go with priority 0. 201 */ 202 prio = delta_max - i; 203 map->aio.cblocks[i].aio_reqprio = prio >= 0 ? prio : 0; 204 } 205 } 206 207 return 0; 208 } 209 210 static void perf_mmap__aio_munmap(struct mmap *map) 211 { 212 int i; 213 214 for (i = 0; i < map->aio.nr_cblocks; ++i) 215 perf_mmap__aio_free(map, i); 216 if (map->aio.data) 217 zfree(&map->aio.data); 218 zfree(&map->aio.cblocks); 219 zfree(&map->aio.aiocb); 220 } 221 #else /* !HAVE_AIO_SUPPORT */ 222 static int perf_mmap__aio_enabled(struct mmap *map __maybe_unused) 223 { 224 return 0; 225 } 226 227 static int perf_mmap__aio_mmap(struct mmap *map __maybe_unused, 228 struct mmap_params *mp __maybe_unused) 229 { 230 return 0; 231 } 232 233 static void perf_mmap__aio_munmap(struct mmap *map __maybe_unused) 234 { 235 } 236 #endif 237 238 void mmap__munmap(struct mmap *map) 239 { 240 bitmap_free(map->affinity_mask.bits); 241 map->affinity_mask.bits = NULL; 242 map->affinity_mask.nbits = 0; 243 244 zstd_fini(&map->zstd_data); 245 246 perf_mmap__aio_munmap(map); 247 if (map->data != NULL) { 248 munmap(map->data, mmap__mmap_len(map)); 249 map->data = NULL; 250 } 251 auxtrace_mmap__munmap(&map->auxtrace_mmap); 252 } 253 254 static void build_node_mask(int node, struct mmap_cpu_mask *mask) 255 { 256 int idx, nr_cpus; 257 struct perf_cpu cpu; 258 struct perf_cpu_map *cpu_map = cpu_map__online(); 259 260 if (!cpu_map) 261 return; 262 263 nr_cpus = perf_cpu_map__nr(cpu_map); 264 for (idx = 0; idx < nr_cpus; idx++) { 265 cpu = perf_cpu_map__cpu(cpu_map, idx); /* map c index to online cpu index */ 266 if (cpu__get_node(cpu) == node) 267 __set_bit(cpu.cpu, mask->bits); 268 } 269 perf_cpu_map__put(cpu_map); 270 } 271 272 static int perf_mmap__setup_affinity_mask(struct mmap *map, struct mmap_params *mp) 273 { 274 map->affinity_mask.nbits = cpu__max_cpu().cpu; 275 map->affinity_mask.bits = bitmap_zalloc(map->affinity_mask.nbits); 276 if (!map->affinity_mask.bits) 277 return -1; 278 279 if (mp->affinity == PERF_AFFINITY_NODE && cpu__max_node() > 1) 280 build_node_mask(cpu__get_node(map->core.cpu), &map->affinity_mask); 281 else if (mp->affinity == PERF_AFFINITY_CPU) 282 __set_bit(map->core.cpu.cpu, map->affinity_mask.bits); 283 284 return 0; 285 } 286 287 int mmap__mmap(struct mmap *map, struct mmap_params *mp, int fd, struct perf_cpu cpu) 288 { 289 if (perf_mmap__mmap(&map->core, &mp->core, fd, cpu)) { 290 pr_debug2("failed to mmap perf event ring buffer, error %d\n", 291 errno); 292 return -1; 293 } 294 295 if (mp->affinity != PERF_AFFINITY_SYS && 296 perf_mmap__setup_affinity_mask(map, mp)) { 297 pr_debug2("failed to alloc mmap affinity mask, error %d\n", 298 errno); 299 return -1; 300 } 301 302 if (verbose == 2) 303 mmap_cpu_mask__scnprintf(&map->affinity_mask, "mmap"); 304 305 map->core.flush = mp->flush; 306 307 if (zstd_init(&map->zstd_data, mp->comp_level)) { 308 pr_debug2("failed to init mmap compressor, error %d\n", errno); 309 return -1; 310 } 311 312 if (mp->comp_level && !perf_mmap__aio_enabled(map)) { 313 map->data = mmap(NULL, mmap__mmap_len(map), PROT_READ|PROT_WRITE, 314 MAP_PRIVATE|MAP_ANONYMOUS, 0, 0); 315 if (map->data == MAP_FAILED) { 316 pr_debug2("failed to mmap data buffer, error %d\n", 317 errno); 318 map->data = NULL; 319 return -1; 320 } 321 } 322 323 if (auxtrace_mmap__mmap(&map->auxtrace_mmap, 324 &mp->auxtrace_mp, map->core.base, fd)) 325 return -1; 326 327 return perf_mmap__aio_mmap(map, mp); 328 } 329 330 int perf_mmap__push(struct mmap *md, void *to, 331 int push(struct mmap *map, void *to, void *buf, size_t size)) 332 { 333 u64 head = perf_mmap__read_head(&md->core); 334 unsigned char *data = md->core.base + page_size; 335 unsigned long size; 336 void *buf; 337 int rc = 0; 338 339 rc = perf_mmap__read_init(&md->core); 340 if (rc < 0) 341 return (rc == -EAGAIN) ? 1 : -1; 342 343 size = md->core.end - md->core.start; 344 345 if ((md->core.start & md->core.mask) + size != (md->core.end & md->core.mask)) { 346 buf = &data[md->core.start & md->core.mask]; 347 size = md->core.mask + 1 - (md->core.start & md->core.mask); 348 md->core.start += size; 349 350 if (push(md, to, buf, size) < 0) { 351 rc = -1; 352 goto out; 353 } 354 } 355 356 buf = &data[md->core.start & md->core.mask]; 357 size = md->core.end - md->core.start; 358 md->core.start += size; 359 360 if (push(md, to, buf, size) < 0) { 361 rc = -1; 362 goto out; 363 } 364 365 md->core.prev = head; 366 perf_mmap__consume(&md->core); 367 out: 368 return rc; 369 } 370