191007045SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 216958497SArnaldo Carvalho de Melo /* 316958497SArnaldo Carvalho de Melo * Copyright (C) 2011-2017, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com> 416958497SArnaldo Carvalho de Melo * 516958497SArnaldo Carvalho de Melo * Parts came from evlist.c builtin-{top,stat,record}.c, see those files for further 616958497SArnaldo Carvalho de Melo * copyright notes. 716958497SArnaldo Carvalho de Melo */ 816958497SArnaldo Carvalho de Melo 916958497SArnaldo Carvalho de Melo #include <sys/mman.h> 1073c17d81SArnaldo Carvalho de Melo #include <inttypes.h> 1173c17d81SArnaldo Carvalho de Melo #include <asm/bug.h> 127f7c536fSArnaldo Carvalho de Melo #include <linux/zalloc.h> 13f2a39fe8SArnaldo Carvalho de Melo #include <stdlib.h> 14f2a39fe8SArnaldo Carvalho de Melo #include <string.h> 157634d533SArnaldo Carvalho de Melo #include <unistd.h> // sysconf() 167728fa0cSJiri Olsa #include <perf/mmap.h> 17c44a8b44SAlexey Budankov #ifdef HAVE_LIBNUMA_SUPPORT 18c44a8b44SAlexey Budankov #include <numaif.h> 19c44a8b44SAlexey Budankov #endif 20f2a39fe8SArnaldo Carvalho de Melo #include "cpumap.h" 2173c17d81SArnaldo Carvalho de Melo #include "debug.h" 2216958497SArnaldo Carvalho de Melo #include "event.h" 2316958497SArnaldo Carvalho de Melo #include "mmap.h" 24c1a604dfSArnaldo Carvalho de Melo #include "../perf.h" 2520f2be1dSJiri Olsa #include <internal/lib.h> /* page_size */ 269c080c02SAlexey Budankov #include <linux/bitmap.h> 279c080c02SAlexey Budankov 289c080c02SAlexey Budankov #define MASK_SIZE 1023 299c080c02SAlexey Budankov void mmap_cpu_mask__scnprintf(struct mmap_cpu_mask *mask, const char *tag) 309c080c02SAlexey Budankov { 319c080c02SAlexey Budankov char buf[MASK_SIZE + 1]; 329c080c02SAlexey Budankov size_t len; 339c080c02SAlexey Budankov 349c080c02SAlexey Budankov len = bitmap_scnprintf(mask->bits, mask->nbits, buf, MASK_SIZE); 359c080c02SAlexey Budankov buf[len] = '\0'; 369c080c02SAlexey Budankov pr_debug("%p: %s mask[%zd]: %s\n", mask, tag, mask->nbits, buf); 379c080c02SAlexey Budankov } 3816958497SArnaldo Carvalho de Melo 39bf59b305SJiri Olsa size_t mmap__mmap_len(struct mmap *map) 4016958497SArnaldo Carvalho de Melo { 41bf59b305SJiri Olsa return perf_mmap__mmap_len(&map->core); 4216958497SArnaldo Carvalho de Melo } 4316958497SArnaldo Carvalho de Melo 4416958497SArnaldo Carvalho de Melo int __weak auxtrace_mmap__mmap(struct auxtrace_mmap *mm __maybe_unused, 4516958497SArnaldo Carvalho de Melo struct auxtrace_mmap_params *mp __maybe_unused, 4616958497SArnaldo Carvalho de Melo void *userpg __maybe_unused, 4716958497SArnaldo Carvalho de Melo int fd __maybe_unused) 4816958497SArnaldo Carvalho de Melo { 4916958497SArnaldo Carvalho de Melo return 0; 5016958497SArnaldo Carvalho de Melo } 5116958497SArnaldo Carvalho de Melo 5216958497SArnaldo Carvalho de Melo void __weak auxtrace_mmap__munmap(struct auxtrace_mmap *mm __maybe_unused) 5316958497SArnaldo Carvalho de Melo { 5416958497SArnaldo Carvalho de Melo } 5516958497SArnaldo Carvalho de Melo 5616958497SArnaldo Carvalho de Melo void __weak auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp __maybe_unused, 5716958497SArnaldo Carvalho de Melo off_t auxtrace_offset __maybe_unused, 5816958497SArnaldo Carvalho de Melo unsigned int auxtrace_pages __maybe_unused, 5916958497SArnaldo Carvalho de Melo bool auxtrace_overwrite __maybe_unused) 6016958497SArnaldo Carvalho de Melo { 6116958497SArnaldo Carvalho de Melo } 6216958497SArnaldo Carvalho de Melo 6316958497SArnaldo Carvalho de Melo void __weak auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp __maybe_unused, 6463503dbaSJiri Olsa struct evlist *evlist __maybe_unused, 65d01508f2SAdrian Hunter struct evsel *evsel __maybe_unused, 6684bd5abaSAdrian Hunter int idx __maybe_unused) 6716958497SArnaldo Carvalho de Melo { 6816958497SArnaldo Carvalho de Melo } 6916958497SArnaldo Carvalho de Melo 700b773831SAlexey Budankov #ifdef HAVE_AIO_SUPPORT 71a5830532SJiri Olsa static int perf_mmap__aio_enabled(struct mmap *map) 7251255a8aSAlexey Budankov { 7351255a8aSAlexey Budankov return map->aio.nr_cblocks > 0; 7451255a8aSAlexey Budankov } 75c44a8b44SAlexey Budankov 76c44a8b44SAlexey Budankov #ifdef HAVE_LIBNUMA_SUPPORT 77a5830532SJiri Olsa static int perf_mmap__aio_alloc(struct mmap *map, int idx) 78c44a8b44SAlexey Budankov { 79bf59b305SJiri Olsa map->aio.data[idx] = mmap(NULL, mmap__mmap_len(map), PROT_READ|PROT_WRITE, 80c44a8b44SAlexey Budankov MAP_PRIVATE|MAP_ANONYMOUS, 0, 0); 81c44a8b44SAlexey Budankov if (map->aio.data[idx] == MAP_FAILED) { 82c44a8b44SAlexey Budankov map->aio.data[idx] = NULL; 83c44a8b44SAlexey Budankov return -1; 84c44a8b44SAlexey Budankov } 85c44a8b44SAlexey Budankov 86c44a8b44SAlexey Budankov return 0; 87c44a8b44SAlexey Budankov } 88c44a8b44SAlexey Budankov 89a5830532SJiri Olsa static void perf_mmap__aio_free(struct mmap *map, int idx) 90c44a8b44SAlexey Budankov { 91c44a8b44SAlexey Budankov if (map->aio.data[idx]) { 92bf59b305SJiri Olsa munmap(map->aio.data[idx], mmap__mmap_len(map)); 93c44a8b44SAlexey Budankov map->aio.data[idx] = NULL; 94c44a8b44SAlexey Budankov } 95c44a8b44SAlexey Budankov } 96c44a8b44SAlexey Budankov 976d18804bSIan Rogers static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, int affinity) 98c44a8b44SAlexey Budankov { 99c44a8b44SAlexey Budankov void *data; 100c44a8b44SAlexey Budankov size_t mmap_len; 10144d462acSAlexey Budankov unsigned long *node_mask; 10244d462acSAlexey Budankov unsigned long node_index; 10344d462acSAlexey Budankov int err = 0; 104c44a8b44SAlexey Budankov 105c44a8b44SAlexey Budankov if (affinity != PERF_AFFINITY_SYS && cpu__max_node() > 1) { 106c44a8b44SAlexey Budankov data = map->aio.data[idx]; 107bf59b305SJiri Olsa mmap_len = mmap__mmap_len(map); 10844d462acSAlexey Budankov node_index = cpu__get_node(cpu); 1097fc5b571SAndy Shevchenko node_mask = bitmap_zalloc(node_index + 1); 11044d462acSAlexey Budankov if (!node_mask) { 11144d462acSAlexey Budankov pr_err("Failed to allocate node mask for mbind: error %m\n"); 112c44a8b44SAlexey Budankov return -1; 113c44a8b44SAlexey Budankov } 11475d7ba32SSean Christopherson __set_bit(node_index, node_mask); 11544d462acSAlexey Budankov if (mbind(data, mmap_len, MPOL_BIND, node_mask, node_index + 1 + 1, 0)) { 11644d462acSAlexey Budankov pr_err("Failed to bind [%p-%p] AIO buffer to node %lu: error %m\n", 11744d462acSAlexey Budankov data, data + mmap_len, node_index); 11844d462acSAlexey Budankov err = -1; 11944d462acSAlexey Budankov } 12044d462acSAlexey Budankov bitmap_free(node_mask); 121c44a8b44SAlexey Budankov } 122c44a8b44SAlexey Budankov 12344d462acSAlexey Budankov return err; 124c44a8b44SAlexey Budankov } 12551255a8aSAlexey Budankov #else /* !HAVE_LIBNUMA_SUPPORT */ 126a5830532SJiri Olsa static int perf_mmap__aio_alloc(struct mmap *map, int idx) 127c44a8b44SAlexey Budankov { 128bf59b305SJiri Olsa map->aio.data[idx] = malloc(mmap__mmap_len(map)); 129c44a8b44SAlexey Budankov if (map->aio.data[idx] == NULL) 130c44a8b44SAlexey Budankov return -1; 131c44a8b44SAlexey Budankov 132c44a8b44SAlexey Budankov return 0; 133c44a8b44SAlexey Budankov } 134c44a8b44SAlexey Budankov 135a5830532SJiri Olsa static void perf_mmap__aio_free(struct mmap *map, int idx) 136c44a8b44SAlexey Budankov { 137c44a8b44SAlexey Budankov zfree(&(map->aio.data[idx])); 138c44a8b44SAlexey Budankov } 139c44a8b44SAlexey Budankov 140a5830532SJiri Olsa static int perf_mmap__aio_bind(struct mmap *map __maybe_unused, int idx __maybe_unused, 1416d18804bSIan Rogers struct perf_cpu cpu __maybe_unused, int affinity __maybe_unused) 142c44a8b44SAlexey Budankov { 143c44a8b44SAlexey Budankov return 0; 144c44a8b44SAlexey Budankov } 145c44a8b44SAlexey Budankov #endif 146c44a8b44SAlexey Budankov 147a5830532SJiri Olsa static int perf_mmap__aio_mmap(struct mmap *map, struct mmap_params *mp) 1480b773831SAlexey Budankov { 149c44a8b44SAlexey Budankov int delta_max, i, prio, ret; 1500b773831SAlexey Budankov 151d3d1af6fSAlexey Budankov map->aio.nr_cblocks = mp->nr_cblocks; 152d3d1af6fSAlexey Budankov if (map->aio.nr_cblocks) { 15393f20c0fSAlexey Budankov map->aio.aiocb = calloc(map->aio.nr_cblocks, sizeof(struct aiocb *)); 15493f20c0fSAlexey Budankov if (!map->aio.aiocb) { 15593f20c0fSAlexey Budankov pr_debug2("failed to allocate aiocb for data buffer, error %m\n"); 15693f20c0fSAlexey Budankov return -1; 15793f20c0fSAlexey Budankov } 15893f20c0fSAlexey Budankov map->aio.cblocks = calloc(map->aio.nr_cblocks, sizeof(struct aiocb)); 15993f20c0fSAlexey Budankov if (!map->aio.cblocks) { 16093f20c0fSAlexey Budankov pr_debug2("failed to allocate cblocks for data buffer, error %m\n"); 16193f20c0fSAlexey Budankov return -1; 16293f20c0fSAlexey Budankov } 16393f20c0fSAlexey Budankov map->aio.data = calloc(map->aio.nr_cblocks, sizeof(void *)); 1640b773831SAlexey Budankov if (!map->aio.data) { 1650b773831SAlexey Budankov pr_debug2("failed to allocate data buffer, error %m\n"); 1660b773831SAlexey Budankov return -1; 1670b773831SAlexey Budankov } 16893f20c0fSAlexey Budankov delta_max = sysconf(_SC_AIO_PRIO_DELTA_MAX); 16993f20c0fSAlexey Budankov for (i = 0; i < map->aio.nr_cblocks; ++i) { 170c44a8b44SAlexey Budankov ret = perf_mmap__aio_alloc(map, i); 171c44a8b44SAlexey Budankov if (ret == -1) { 17293f20c0fSAlexey Budankov pr_debug2("failed to allocate data buffer area, error %m"); 17393f20c0fSAlexey Budankov return -1; 17493f20c0fSAlexey Budankov } 17556a94706SJiri Olsa ret = perf_mmap__aio_bind(map, i, map->core.cpu, mp->affinity); 176c44a8b44SAlexey Budankov if (ret == -1) 177c44a8b44SAlexey Budankov return -1; 1780b773831SAlexey Budankov /* 1790b773831SAlexey Budankov * Use cblock.aio_fildes value different from -1 1800b773831SAlexey Budankov * to denote started aio write operation on the 1810b773831SAlexey Budankov * cblock so it requires explicit record__aio_sync() 1820b773831SAlexey Budankov * call prior the cblock may be reused again. 1830b773831SAlexey Budankov */ 18493f20c0fSAlexey Budankov map->aio.cblocks[i].aio_fildes = -1; 1850b773831SAlexey Budankov /* 18693f20c0fSAlexey Budankov * Allocate cblocks with priority delta to have 18793f20c0fSAlexey Budankov * faster aio write system calls because queued requests 18893f20c0fSAlexey Budankov * are kept in separate per-prio queues and adding 18993f20c0fSAlexey Budankov * a new request will iterate thru shorter per-prio 19093f20c0fSAlexey Budankov * list. Blocks with numbers higher than 19193f20c0fSAlexey Budankov * _SC_AIO_PRIO_DELTA_MAX go with priority 0. 1920b773831SAlexey Budankov */ 19393f20c0fSAlexey Budankov prio = delta_max - i; 19493f20c0fSAlexey Budankov map->aio.cblocks[i].aio_reqprio = prio >= 0 ? prio : 0; 19593f20c0fSAlexey Budankov } 1960b773831SAlexey Budankov } 1970b773831SAlexey Budankov 1980b773831SAlexey Budankov return 0; 1990b773831SAlexey Budankov } 2000b773831SAlexey Budankov 201a5830532SJiri Olsa static void perf_mmap__aio_munmap(struct mmap *map) 2020b773831SAlexey Budankov { 203c8dd6ee5SAlexey Budankov int i; 204c8dd6ee5SAlexey Budankov 205c8dd6ee5SAlexey Budankov for (i = 0; i < map->aio.nr_cblocks; ++i) 206c44a8b44SAlexey Budankov perf_mmap__aio_free(map, i); 2070b773831SAlexey Budankov if (map->aio.data) 2080b773831SAlexey Budankov zfree(&map->aio.data); 209c8dd6ee5SAlexey Budankov zfree(&map->aio.cblocks); 210c8dd6ee5SAlexey Budankov zfree(&map->aio.aiocb); 2110b773831SAlexey Budankov } 21251255a8aSAlexey Budankov #else /* !HAVE_AIO_SUPPORT */ 213a5830532SJiri Olsa static int perf_mmap__aio_enabled(struct mmap *map __maybe_unused) 21451255a8aSAlexey Budankov { 21551255a8aSAlexey Budankov return 0; 21651255a8aSAlexey Budankov } 21751255a8aSAlexey Budankov 218a5830532SJiri Olsa static int perf_mmap__aio_mmap(struct mmap *map __maybe_unused, 2190b773831SAlexey Budankov struct mmap_params *mp __maybe_unused) 2200b773831SAlexey Budankov { 2210b773831SAlexey Budankov return 0; 2220b773831SAlexey Budankov } 2230b773831SAlexey Budankov 224a5830532SJiri Olsa static void perf_mmap__aio_munmap(struct mmap *map __maybe_unused) 2250b773831SAlexey Budankov { 2260b773831SAlexey Budankov } 2270b773831SAlexey Budankov #endif 2280b773831SAlexey Budankov 22959d7ea62SJiri Olsa void mmap__munmap(struct mmap *map) 23016958497SArnaldo Carvalho de Melo { 2318384a260SAlexey Budankov bitmap_free(map->affinity_mask.bits); 2328384a260SAlexey Budankov 23375f5f1fcSAlexey Bayduraev zstd_fini(&map->zstd_data); 23475f5f1fcSAlexey Bayduraev 2350b773831SAlexey Budankov perf_mmap__aio_munmap(map); 23651255a8aSAlexey Budankov if (map->data != NULL) { 237bf59b305SJiri Olsa munmap(map->data, mmap__mmap_len(map)); 23851255a8aSAlexey Budankov map->data = NULL; 23951255a8aSAlexey Budankov } 24016958497SArnaldo Carvalho de Melo auxtrace_mmap__munmap(&map->auxtrace_mmap); 24116958497SArnaldo Carvalho de Melo } 24216958497SArnaldo Carvalho de Melo 2438384a260SAlexey Budankov static void build_node_mask(int node, struct mmap_cpu_mask *mask) 244f13de660SAlexey Budankov { 2456d18804bSIan Rogers int idx, nr_cpus; 2466d18804bSIan Rogers struct perf_cpu cpu; 247f854839bSJiri Olsa const struct perf_cpu_map *cpu_map = NULL; 248f13de660SAlexey Budankov 249f13de660SAlexey Budankov cpu_map = cpu_map__online(); 250f13de660SAlexey Budankov if (!cpu_map) 251f13de660SAlexey Budankov return; 252f13de660SAlexey Budankov 2536549cd8fSJiri Olsa nr_cpus = perf_cpu_map__nr(cpu_map); 2546d18804bSIan Rogers for (idx = 0; idx < nr_cpus; idx++) { 25544028699SIan Rogers cpu = perf_cpu_map__cpu(cpu_map, idx); /* map c index to online cpu index */ 256f13de660SAlexey Budankov if (cpu__get_node(cpu) == node) 25775d7ba32SSean Christopherson __set_bit(cpu.cpu, mask->bits); 258f13de660SAlexey Budankov } 259f13de660SAlexey Budankov } 260f13de660SAlexey Budankov 2618384a260SAlexey Budankov static int perf_mmap__setup_affinity_mask(struct mmap *map, struct mmap_params *mp) 262f13de660SAlexey Budankov { 2636d18804bSIan Rogers map->affinity_mask.nbits = cpu__max_cpu().cpu; 2647fc5b571SAndy Shevchenko map->affinity_mask.bits = bitmap_zalloc(map->affinity_mask.nbits); 2658384a260SAlexey Budankov if (!map->affinity_mask.bits) 2668384a260SAlexey Budankov return -1; 2678384a260SAlexey Budankov 268f13de660SAlexey Budankov if (mp->affinity == PERF_AFFINITY_NODE && cpu__max_node() > 1) 26956a94706SJiri Olsa build_node_mask(cpu__get_node(map->core.cpu), &map->affinity_mask); 270f13de660SAlexey Budankov else if (mp->affinity == PERF_AFFINITY_CPU) 27175d7ba32SSean Christopherson __set_bit(map->core.cpu.cpu, map->affinity_mask.bits); 2728384a260SAlexey Budankov 2738384a260SAlexey Budankov return 0; 274f13de660SAlexey Budankov } 275f13de660SAlexey Budankov 2766d18804bSIan Rogers int mmap__mmap(struct mmap *map, struct mmap_params *mp, int fd, struct perf_cpu cpu) 27716958497SArnaldo Carvalho de Melo { 27832c261c0SJiri Olsa if (perf_mmap__mmap(&map->core, &mp->core, fd, cpu)) { 27916958497SArnaldo Carvalho de Melo pr_debug2("failed to mmap perf event ring buffer, error %d\n", 28016958497SArnaldo Carvalho de Melo errno); 28116958497SArnaldo Carvalho de Melo return -1; 28216958497SArnaldo Carvalho de Melo } 28316958497SArnaldo Carvalho de Melo 2848384a260SAlexey Budankov if (mp->affinity != PERF_AFFINITY_SYS && 2858384a260SAlexey Budankov perf_mmap__setup_affinity_mask(map, mp)) { 2868384a260SAlexey Budankov pr_debug2("failed to alloc mmap affinity mask, error %d\n", 2878384a260SAlexey Budankov errno); 2888384a260SAlexey Budankov return -1; 2898384a260SAlexey Budankov } 2908384a260SAlexey Budankov 2918384a260SAlexey Budankov if (verbose == 2) 2928384a260SAlexey Budankov mmap_cpu_mask__scnprintf(&map->affinity_mask, "mmap"); 2939d2ed645SAlexey Budankov 29465aa2e6bSJiri Olsa map->core.flush = mp->flush; 295470530bbSAlexey Budankov 296*5940a20aSIan Rogers if (zstd_init(&map->zstd_data, mp->comp_level)) { 297b47f18d8SColin Ian King pr_debug2("failed to init mmap compressor, error %d\n", errno); 29875f5f1fcSAlexey Bayduraev return -1; 29975f5f1fcSAlexey Bayduraev } 30051255a8aSAlexey Budankov 301*5940a20aSIan Rogers if (mp->comp_level && !perf_mmap__aio_enabled(map)) { 302bf59b305SJiri Olsa map->data = mmap(NULL, mmap__mmap_len(map), PROT_READ|PROT_WRITE, 30351255a8aSAlexey Budankov MAP_PRIVATE|MAP_ANONYMOUS, 0, 0); 30451255a8aSAlexey Budankov if (map->data == MAP_FAILED) { 30551255a8aSAlexey Budankov pr_debug2("failed to mmap data buffer, error %d\n", 30651255a8aSAlexey Budankov errno); 30751255a8aSAlexey Budankov map->data = NULL; 30851255a8aSAlexey Budankov return -1; 30951255a8aSAlexey Budankov } 31051255a8aSAlexey Budankov } 31151255a8aSAlexey Budankov 31216958497SArnaldo Carvalho de Melo if (auxtrace_mmap__mmap(&map->auxtrace_mmap, 313547740f7SJiri Olsa &mp->auxtrace_mp, map->core.base, fd)) 31416958497SArnaldo Carvalho de Melo return -1; 31516958497SArnaldo Carvalho de Melo 3160b773831SAlexey Budankov return perf_mmap__aio_mmap(map, mp); 31716958497SArnaldo Carvalho de Melo } 31873c17d81SArnaldo Carvalho de Melo 319a5830532SJiri Olsa int perf_mmap__push(struct mmap *md, void *to, 320a5830532SJiri Olsa int push(struct mmap *map, void *to, void *buf, size_t size)) 3218872481bSKan Liang { 3227728fa0cSJiri Olsa u64 head = perf_mmap__read_head(&md->core); 323547740f7SJiri Olsa unsigned char *data = md->core.base + page_size; 3248872481bSKan Liang unsigned long size; 3258872481bSKan Liang void *buf; 3268872481bSKan Liang int rc = 0; 3278872481bSKan Liang 3287c4d4182SJiri Olsa rc = perf_mmap__read_init(&md->core); 329189f2cc9SKan Liang if (rc < 0) 330ef781128SAlexey Budankov return (rc == -EAGAIN) ? 1 : -1; 3318872481bSKan Liang 332ebe4d72bSJiri Olsa size = md->core.end - md->core.start; 333dc6c35c6SKan Liang 334ebe4d72bSJiri Olsa if ((md->core.start & md->core.mask) + size != (md->core.end & md->core.mask)) { 335ebe4d72bSJiri Olsa buf = &data[md->core.start & md->core.mask]; 336ebe4d72bSJiri Olsa size = md->core.mask + 1 - (md->core.start & md->core.mask); 337ebe4d72bSJiri Olsa md->core.start += size; 33873c17d81SArnaldo Carvalho de Melo 339ded2b8feSJiri Olsa if (push(md, to, buf, size) < 0) { 34073c17d81SArnaldo Carvalho de Melo rc = -1; 34173c17d81SArnaldo Carvalho de Melo goto out; 34273c17d81SArnaldo Carvalho de Melo } 34373c17d81SArnaldo Carvalho de Melo } 34473c17d81SArnaldo Carvalho de Melo 345ebe4d72bSJiri Olsa buf = &data[md->core.start & md->core.mask]; 346ebe4d72bSJiri Olsa size = md->core.end - md->core.start; 347ebe4d72bSJiri Olsa md->core.start += size; 34873c17d81SArnaldo Carvalho de Melo 349ded2b8feSJiri Olsa if (push(md, to, buf, size) < 0) { 35073c17d81SArnaldo Carvalho de Melo rc = -1; 35173c17d81SArnaldo Carvalho de Melo goto out; 35273c17d81SArnaldo Carvalho de Melo } 35373c17d81SArnaldo Carvalho de Melo 354ebe4d72bSJiri Olsa md->core.prev = head; 3557728fa0cSJiri Olsa perf_mmap__consume(&md->core); 35673c17d81SArnaldo Carvalho de Melo out: 35773c17d81SArnaldo Carvalho de Melo return rc; 35873c17d81SArnaldo Carvalho de Melo } 3596bd006c6SRiccardo Mancini 3606bd006c6SRiccardo Mancini int mmap_cpu_mask__duplicate(struct mmap_cpu_mask *original, struct mmap_cpu_mask *clone) 3616bd006c6SRiccardo Mancini { 3626bd006c6SRiccardo Mancini clone->nbits = original->nbits; 3636bd006c6SRiccardo Mancini clone->bits = bitmap_zalloc(original->nbits); 3646bd006c6SRiccardo Mancini if (!clone->bits) 3656bd006c6SRiccardo Mancini return -ENOMEM; 3666bd006c6SRiccardo Mancini 3676bd006c6SRiccardo Mancini memcpy(clone->bits, original->bits, MMAP_CPU_MASK_BYTES(original)); 3686bd006c6SRiccardo Mancini return 0; 3696bd006c6SRiccardo Mancini } 370