1 // SPDX-License-Identifier: GPL-2.0
3  * mem-memcpy.c
11 #include "../perf-sys.h"
12 #include <subcmd/parse-options.h>
17 #include "mem-memcpy-arch.h"
18 #include "mem-memset-arch.h"
51 		    "Specify page-size for mapping memory buffers. "
54 	OPT_STRING('f', "function", &function_str, "all",
55 		    "Specify the function to run, \"all\" runs all available functions, \"help\" lists them"),
68 		    "Specify the chunk-size for each invocation. "
89 	const struct function *functions;
90 	int (*do_op)(const struct function *r, struct bench_params *p,
105 struct function {  struct
126 	cycles_fd = sys_perf_event_open(&cycle_attr, getpid(), -1, -1, perf_event_open_cloexec_flag());  in init_cycles()
130 		return -1;  in init_cycles()
150 		t->cycles = get_cycles();  in clock_get()
152 		BUG_ON(gettimeofday(&t->tv, NULL));  in clock_get()
160 		t.cycles = e->cycles - s->cycles;  in clock_diff()
162 		timersub(&e->tv, &s->tv, &t.tv);  in clock_diff()
170 		a->cycles += b->cycles;  in clock_accum()
172 		timeradd(&a->tv, &b->tv, &a->tv);  in clock_accum()
177 	return (double)ts->tv_sec + (double)ts->tv_usec / (double)USEC_PER_SEC;  in timeval2double()
194 	const struct function *r = &info->functions[r_idx];  in __bench_mem_function()
199 	printf("# function '%s' (%s)\n", r->name, r->desc);  in __bench_mem_function()
201 	if (r->fn.init && r->fn.init(info, p, &src, &dst))  in __bench_mem_function()
207 	if (info->do_op(r, p, src, dst, &rt))  in __bench_mem_function()
213 			printf(" %14lf cycles/byte\n", (double)rt.cycles/(double)p->size_total);  in __bench_mem_function()
215 			result_bps = (double)p->size_total/timeval2double(&rt.tv);  in __bench_mem_function()
222 			printf("%lf\n", (double)rt.cycles/(double)p->size_total);  in __bench_mem_function()
224 			result_bps = (double)p->size_total/timeval2double(&rt.tv);  in __bench_mem_function()
236 	if (r->fn.fini) r->fn.fini(info, p, &src, &dst);  in __bench_mem_function()
239 	printf("# Memory allocation failed - maybe size (%s) %s?\n", size_str,  in __bench_mem_function()
240 			p->page_shift != PAGE_SHIFT_4KB ? "has insufficient hugepages" : "is too large");  in __bench_mem_function()
250 	argc = parse_options(argc, argv, info->options, info->usage, 0);  in bench_mem_common()
281 		fprintf(stderr, "Invalid page-size:%s\n", page_size_str);  in bench_mem_common()
289 		for (i = 0; info->functions[i].name; i++)  in bench_mem_common()
294 	for (i = 0; info->functions[i].name; i++) {  in bench_mem_common()
295 		if (!strcmp(info->functions[i].name, function_str))  in bench_mem_common()
298 	if (!info->functions[i].name) {  in bench_mem_common()
300 			printf("Unknown function: %s\n", function_str);  in bench_mem_common()
302 		for (i = 0; info->functions[i].name; i++) {  in bench_mem_common()
304 			       info->functions[i].name, info->functions[i].desc);  in bench_mem_common()
326 static int do_memcpy(const struct function *r, struct bench_params *p,  in do_memcpy()
330 	memcpy_t fn = r->fn.memcpy;  in do_memcpy()
332 	memcpy_prefault(fn, p->size, src, dst);  in do_memcpy()
335 	for (unsigned int i = 0; i < p->nr_loops; ++i)  in do_memcpy()
336 		for (size_t off = 0; off < p->size; off += p->chunk_size)  in do_memcpy()  local
337 			fn(dst + off, src + off, min(p->chunk_size, p->size - off));  in do_memcpy()
370 	*dst = bench_mmap(p->size, true, p->page_shift);  in mem_alloc()
373 	if (info->alloc_src) {  in mem_alloc()
374 		*src = bench_mmap(p->size, true, p->page_shift);  in mem_alloc()
385 	bench_munmap(*dst, p->size);  in mem_free()
386 	bench_munmap(*src, p->size);  in mem_free()
391 struct function memcpy_functions[] = {
401 # include "mem-memcpy-x86-64-asm-def.h"
426 static int do_memset(const struct function *r, struct bench_params *p,  in do_memset()
430 	memset_t fn = r->fn.memset;  in do_memset()
436 	fn(dst, -1, p->size);  in do_memset()
439 	for (unsigned int i = 0; i < p->nr_loops; ++i)  in do_memset()
440 		for (size_t off = 0; off < p->size; off += p->chunk_size)  in do_memset()  local
441 			fn(dst + off, i, min(p->chunk_size, p->size - off));  in do_memset()
454 static const struct function memset_functions[] = {
464 # include "mem-memset-x86-64-asm-def.h"
497 static int do_mmap(const struct function *r, struct bench_params *p,  in do_mmap()
502 	mmap_op_t fn = r->fn.mmap_op;  in do_mmap()
503 	bool populate = strcmp(r->name, "populate") == 0;  in do_mmap()
505 	if (p->seed)  in do_mmap()
506 		srand(p->seed);  in do_mmap()
508 	for (unsigned int i = 0; i < p->nr_loops; i++) {  in do_mmap()
510 		dst = bench_mmap(p->size, populate, p->page_shift);  in do_mmap()
514 		fn(dst, p->size, p->page_shift, p->seed);  in do_mmap()
519 		bench_munmap(dst, p->size);  in do_mmap()
524 	printf("# Memory allocation failed - maybe size (%s) %s?\n", size_str,  in do_mmap()
525 			p->page_shift != PAGE_SHIFT_4KB ? "has insufficient hugepages" : "is too large");  in do_mmap()
526 	return -1;  in do_mmap()
534 static const struct function mmap_functions[] = {