xref: /freebsd/contrib/arm-optimized-routines/string/bench/memcpy.c (revision 9f23cbd6cae82fd77edfad7173432fa8dccd0a95)
1 /*
2  * memcpy benchmark.
3  *
4  * Copyright (c) 2020-2022, Arm Limited.
5  * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
6  */
7 
8 #define _GNU_SOURCE
9 #include <stdint.h>
10 #include <stdio.h>
11 #include <string.h>
12 #include <assert.h>
13 #include "stringlib.h"
14 #include "benchlib.h"
15 
16 #define ITERS  5000
17 #define ITERS2 20000000
18 #define ITERS3 200000
19 #define NUM_TESTS 16384
20 #define MIN_SIZE 32768
21 #define MAX_SIZE (1024 * 1024)
22 
23 static uint8_t a[MAX_SIZE + 4096 + 64] __attribute__((__aligned__(64)));
24 static uint8_t b[MAX_SIZE + 4096 + 64] __attribute__((__aligned__(64)));
25 
26 #define F(x) {#x, x},
27 
28 static const struct fun
29 {
30   const char *name;
31   void *(*fun)(void *, const void *, size_t);
32 } funtab[] =
33 {
34 #if __aarch64__
35   F(__memcpy_aarch64)
36 # if __ARM_NEON
37   F(__memcpy_aarch64_simd)
38 # endif
39 # if __ARM_FEATURE_SVE
40   F(__memcpy_aarch64_sve)
41 # endif
42 #elif __arm__
43   F(__memcpy_arm)
44 #endif
45   F(memcpy)
46 #undef F
47   {0, 0}
48 };
49 
50 typedef struct { uint16_t size; uint16_t freq; } freq_data_t;
51 typedef struct { uint8_t align; uint16_t freq; } align_data_t;
52 
53 #define SIZE_NUM 65536
54 #define SIZE_MASK (SIZE_NUM-1)
55 static uint8_t size_arr[SIZE_NUM];
56 
57 /* Frequency data for memcpy of less than 4096 bytes based on SPEC2017.  */
58 static freq_data_t size_freq[] =
59 {
60 {32,22320}, { 16,9554}, {  8,8915}, {152,5327}, {  4,2159}, {292,2035},
61 { 12,1608}, { 24,1343}, {1152,895}, {144, 813}, {884, 733}, {284, 721},
62 {120, 661}, {  2, 649}, {882, 550}, {  5, 475}, {  7, 461}, {108, 460},
63 { 10, 361}, {  9, 361}, {  6, 334}, {  3, 326}, {464, 308}, {2048,303},
64 {  1, 298}, { 64, 250}, { 11, 197}, {296, 194}, { 68, 187}, { 15, 185},
65 {192, 184}, {1764,183}, { 13, 173}, {560, 126}, {160, 115}, {288,  96},
66 {104,  96}, {1144, 83}, { 18,  80}, { 23,  78}, { 40,  77}, { 19,  68},
67 { 48,  63}, { 17,  57}, { 72,  54}, {1280, 51}, { 20,  49}, { 28,  47},
68 { 22,  46}, {640,  45}, { 25,  41}, { 14,  40}, { 56,  37}, { 27,  35},
69 { 35,  33}, {384,  33}, { 29,  32}, { 80,  30}, {4095, 22}, {232,  22},
70 { 36,  19}, {184,  17}, { 21,  17}, {256,  16}, { 44,  15}, { 26,  15},
71 { 31,  14}, { 88,  14}, {176,  13}, { 33,  12}, {1024, 12}, {208,  11},
72 { 62,  11}, {128,  10}, {704,  10}, {324,  10}, { 96,  10}, { 60,   9},
73 {136,   9}, {124,   9}, { 34,   8}, { 30,   8}, {480,   8}, {1344,  8},
74 {273,   7}, {520,   7}, {112,   6}, { 52,   6}, {344,   6}, {336,   6},
75 {504,   5}, {168,   5}, {424,   5}, {  0,   4}, { 76,   3}, {200,   3},
76 {512,   3}, {312,   3}, {240,   3}, {960,   3}, {264,   2}, {672,   2},
77 { 38,   2}, {328,   2}, { 84,   2}, { 39,   2}, {216,   2}, { 42,   2},
78 { 37,   2}, {1608,  2}, { 70,   2}, { 46,   2}, {536,   2}, {280,   1},
79 {248,   1}, { 47,   1}, {1088,  1}, {1288,  1}, {224,   1}, { 41,   1},
80 { 50,   1}, { 49,   1}, {808,   1}, {360,   1}, {440,   1}, { 43,   1},
81 { 45,   1}, { 78,   1}, {968,   1}, {392,   1}, { 54,   1}, { 53,   1},
82 { 59,   1}, {376,   1}, {664,   1}, { 58,   1}, {272,   1}, { 66,   1},
83 {2688,  1}, {472,   1}, {568,   1}, {720,   1}, { 51,   1}, { 63,   1},
84 { 86,   1}, {496,   1}, {776,   1}, { 57,   1}, {680,   1}, {792,   1},
85 {122,   1}, {760,   1}, {824,   1}, {552,   1}, { 67,   1}, {456,   1},
86 {984,   1}, { 74,   1}, {408,   1}, { 75,   1}, { 92,   1}, {576,   1},
87 {116,   1}, { 65,   1}, {117,   1}, { 82,   1}, {352,   1}, { 55,   1},
88 {100,   1}, { 90,   1}, {696,   1}, {111,   1}, {880,   1}, { 79,   1},
89 {488,   1}, { 61,   1}, {114,   1}, { 94,   1}, {1032,  1}, { 98,   1},
90 { 87,   1}, {584,   1}, { 85,   1}, {648,   1}, {0, 0}
91 };
92 
93 #define ALIGN_NUM 1024
94 #define ALIGN_MASK (ALIGN_NUM-1)
95 static uint8_t src_align_arr[ALIGN_NUM];
96 static uint8_t dst_align_arr[ALIGN_NUM];
97 
98 /* Source alignment frequency for memcpy based on SPEC2017.  */
99 static align_data_t src_align_freq[] =
100 {
101   {8, 300}, {16, 292}, {32, 168}, {64, 153}, {4, 79}, {2, 14}, {1, 18}, {0, 0}
102 };
103 
104 static align_data_t dst_align_freq[] =
105 {
106   {8, 265}, {16, 263}, {64, 209}, {32, 174}, {4, 90}, {2, 10}, {1, 13}, {0, 0}
107 };
108 
109 typedef struct
110 {
111   uint64_t src : 24;
112   uint64_t dst : 24;
113   uint64_t len : 16;
114 } copy_t;
115 
116 static copy_t test_arr[NUM_TESTS];
117 
118 typedef char *(*proto_t) (char *, const char *, size_t);
119 
120 static void
121 init_copy_distribution (void)
122 {
123   int i, j, freq, size, n;
124 
125   for (n = i = 0; (freq = size_freq[i].freq) != 0; i++)
126     for (j = 0, size = size_freq[i].size; j < freq; j++)
127       size_arr[n++] = size;
128   assert (n == SIZE_NUM);
129 
130   for (n = i = 0; (freq = src_align_freq[i].freq) != 0; i++)
131     for (j = 0, size = src_align_freq[i].align; j < freq; j++)
132       src_align_arr[n++] = size - 1;
133   assert (n == ALIGN_NUM);
134 
135   for (n = i = 0; (freq = dst_align_freq[i].freq) != 0; i++)
136     for (j = 0, size = dst_align_freq[i].align; j < freq; j++)
137       dst_align_arr[n++] = size - 1;
138   assert (n == ALIGN_NUM);
139 }
140 
141 static size_t
142 init_copies (size_t max_size)
143 {
144   size_t total = 0;
145   /* Create a random set of copies with the given size and alignment
146      distributions.  */
147   for (int i = 0; i < NUM_TESTS; i++)
148     {
149       test_arr[i].dst = (rand32 (0) & (max_size - 1));
150       test_arr[i].dst &= ~dst_align_arr[rand32 (0) & ALIGN_MASK];
151       test_arr[i].src = (rand32 (0) & (max_size - 1));
152       test_arr[i].src &= ~src_align_arr[rand32 (0) & ALIGN_MASK];
153       test_arr[i].len = size_arr[rand32 (0) & SIZE_MASK];
154       total += test_arr[i].len;
155     }
156 
157   return total;
158 }
159 
160 int main (void)
161 {
162   init_copy_distribution ();
163 
164   memset (a, 1, sizeof (a));
165   memset (b, 2, sizeof (b));
166 
167   printf("Random memcpy (bytes/ns):\n");
168   for (int f = 0; funtab[f].name != 0; f++)
169     {
170       size_t total = 0;
171       uint64_t tsum = 0;
172       printf ("%22s ", funtab[f].name);
173       rand32 (0x12345678);
174 
175       for (int size = MIN_SIZE; size <= MAX_SIZE; size *= 2)
176 	{
177 	  size_t copy_size = init_copies (size) * ITERS;
178 
179 	  for (int c = 0; c < NUM_TESTS; c++)
180 	    funtab[f].fun (b + test_arr[c].dst, a + test_arr[c].src,
181 			   test_arr[c].len);
182 
183 	  uint64_t t = clock_get_ns ();
184 	  for (int i = 0; i < ITERS; i++)
185 	    for (int c = 0; c < NUM_TESTS; c++)
186 	      funtab[f].fun (b + test_arr[c].dst, a + test_arr[c].src,
187 			     test_arr[c].len);
188 	  t = clock_get_ns () - t;
189 	  total += copy_size;
190 	  tsum += t;
191 	  printf ("%dK: %.2f ", size / 1024, (double)copy_size / t);
192 	}
193       printf( "avg %.2f\n", (double)total / tsum);
194     }
195 
196   size_t total = 0;
197   uint64_t tsum = 0;
198   printf ("%22s ", "memcpy_call");
199   rand32 (0x12345678);
200 
201   for (int size = MIN_SIZE; size <= MAX_SIZE; size *= 2)
202     {
203       size_t copy_size = init_copies (size) * ITERS;
204 
205       for (int c = 0; c < NUM_TESTS; c++)
206 	memcpy (b + test_arr[c].dst, a + test_arr[c].src, test_arr[c].len);
207 
208       uint64_t t = clock_get_ns ();
209       for (int i = 0; i < ITERS; i++)
210 	for (int c = 0; c < NUM_TESTS; c++)
211 	  memcpy (b + test_arr[c].dst, a + test_arr[c].src, test_arr[c].len);
212       t = clock_get_ns () - t;
213       total += copy_size;
214       tsum += t;
215       printf ("%dK: %.2f ", size / 1024, (double)copy_size / t);
216     }
217   printf( "avg %.2f\n", (double)total / tsum);
218 
219 
220   printf ("\nAligned medium memcpy (bytes/ns):\n");
221   for (int f = 0; funtab[f].name != 0; f++)
222     {
223       printf ("%22s ", funtab[f].name);
224 
225       for (int size = 8; size <= 512; size *= 2)
226 	{
227 	  uint64_t t = clock_get_ns ();
228 	  for (int i = 0; i < ITERS2; i++)
229 	    funtab[f].fun (b, a, size);
230 	  t = clock_get_ns () - t;
231 	  printf ("%dB: %.2f ", size, (double)size * ITERS2 / t);
232 	}
233       printf ("\n");
234     }
235 
236   printf ("%22s ", "memcpy_call");
237   for (int size = 8; size <= 512; size *= 2)
238     {
239       uint64_t t = clock_get_ns ();
240       for (int i = 0; i < ITERS2; i++)
241 	memcpy (b, a, size);
242       t = clock_get_ns () - t;
243       printf ("%dB: %.2f ", size, (double)size * ITERS2 / t);
244     }
245   printf ("\n");
246 
247 
248   printf ("\nUnaligned medium memcpy (bytes/ns):\n");
249   for (int f = 0; funtab[f].name != 0; f++)
250     {
251       printf ("%22s ", funtab[f].name);
252 
253       for (int size = 8; size <= 512; size *= 2)
254 	{
255 	  uint64_t t = clock_get_ns ();
256 	  for (int i = 0; i < ITERS2; i++)
257 	    funtab[f].fun (b + 3, a + 1, size);
258 	  t = clock_get_ns () - t;
259 	  printf ("%dB: %.2f ", size, (double)size * ITERS2 / t);
260 	}
261       printf ("\n");
262     }
263 
264   printf ("%22s ", "memcpy_call");
265   for (int size = 8; size <= 512; size *= 2)
266     {
267       uint64_t t = clock_get_ns ();
268       for (int i = 0; i < ITERS2; i++)
269 	memcpy (b + 3, a + 1, size);
270       t = clock_get_ns () - t;
271       printf ("%dB: %.2f ", size, (double)size * ITERS2 / t);
272     }
273   printf ("\n");
274 
275 
276   printf ("\nLarge memcpy (bytes/ns):\n");
277   for (int f = 0; funtab[f].name != 0; f++)
278     {
279       printf ("%22s ", funtab[f].name);
280 
281       for (int size = 1024; size <= 65536; size *= 2)
282 	{
283 	  uint64_t t = clock_get_ns ();
284 	  for (int i = 0; i < ITERS3; i++)
285 	    funtab[f].fun (b, a, size);
286 	  t = clock_get_ns () - t;
287 	  printf ("%dK: %.2f ", size / 1024, (double)size * ITERS3 / t);
288 	}
289       printf ("\n");
290     }
291 
292   printf ("%22s ", "memcpy_call");
293   for (int size = 1024; size <= 65536; size *= 2)
294     {
295       uint64_t t = clock_get_ns ();
296       for (int i = 0; i < ITERS3; i++)
297 	memcpy (b, a, size);
298       t = clock_get_ns () - t;
299       printf ("%dK: %.2f ", size / 1024, (double)size * ITERS3 / t);
300     }
301   printf ("\n");
302 
303 
304   printf ("\nUnaligned forwards memmove (bytes/ns):\n");
305   for (int f = 0; funtab[f].name != 0; f++)
306     {
307       printf ("%22s ", funtab[f].name);
308 
309       for (int size = 1024; size <= 65536; size *= 2)
310 	{
311 	  uint64_t t = clock_get_ns ();
312 	  for (int i = 0; i < ITERS3; i++)
313 	    funtab[f].fun (a, a + 256 + (i & 31), size);
314 	  t = clock_get_ns () - t;
315 	  printf ("%dK: %.2f ", size / 1024, (double)size * ITERS3 / t);
316 	}
317       printf ("\n");
318     }
319 
320 
321   printf ("\nUnaligned backwards memmove (bytes/ns):\n");
322   for (int f = 0; funtab[f].name != 0; f++)
323     {
324       printf ("%22s ", funtab[f].name);
325 
326       for (int size = 1024; size <= 65536; size *= 2)
327 	{
328 	  uint64_t t = clock_get_ns ();
329 	  for (int i = 0; i < ITERS3; i++)
330 	    funtab[f].fun (a + 256 + (i & 31), a, size);
331 	  t = clock_get_ns () - t;
332 	  printf ("%dK: %.2f ", size / 1024, (double)size * ITERS3 / t);
333 	}
334       printf ("\n");
335     }
336   printf ("\n");
337 
338   return 0;
339 }
340