xref: /linux/crypto/tcrypt.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Quick & dirty crypto testing module.
4  *
5  * This will only exist until we have a better testing mechanism
6  * (e.g. a char device).
7  *
8  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
9  * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
10  * Copyright (c) 2007 Nokia Siemens Networks
11  *
12  * Updated RFC4106 AES-GCM testing.
13  *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
14  *             Adrian Hoban <adrian.hoban@intel.com>
15  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
16  *             Tadeusz Struk (tadeusz.struk@intel.com)
17  *             Copyright (c) 2010, Intel Corporation.
18  */
19 
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 
22 #include <crypto/aead.h>
23 #include <crypto/hash.h>
24 #include <crypto/skcipher.h>
25 #include <linux/err.h>
26 #include <linux/fips.h>
27 #include <linux/init.h>
28 #include <linux/interrupt.h>
29 #include <linux/jiffies.h>
30 #include <linux/kernel.h>
31 #include <linux/module.h>
32 #include <linux/moduleparam.h>
33 #include <linux/scatterlist.h>
34 #include <linux/slab.h>
35 #include <linux/string.h>
36 #include <linux/timex.h>
37 
38 #include "internal.h"
39 #include "tcrypt.h"
40 
41 /*
42  * Need slab memory for testing (size in number of pages).
43  */
44 #define TVMEMSIZE	4
45 
46 /*
47 * Used by test_cipher_speed()
48 */
49 #define ENCRYPT 1
50 #define DECRYPT 0
51 
52 #define MAX_DIGEST_SIZE		64
53 
54 /*
55  * return a string with the driver name
56  */
57 #define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm))
58 
59 /*
60  * Used by test_cipher_speed()
61  */
62 static unsigned int sec;
63 
64 static char *alg;
65 static u32 type;
66 static u32 mask;
67 static int mode;
68 static u32 num_mb = 8;
69 static unsigned int klen;
70 static char *tvmem[TVMEMSIZE];
71 
72 static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 };
73 static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 };
74 
75 #define XBUFSIZE 8
76 #define MAX_IVLEN 32
77 
testmgr_alloc_buf(char * buf[XBUFSIZE])78 static int testmgr_alloc_buf(char *buf[XBUFSIZE])
79 {
80 	int i;
81 
82 	for (i = 0; i < XBUFSIZE; i++) {
83 		buf[i] = (void *)__get_free_page(GFP_KERNEL);
84 		if (!buf[i])
85 			goto err_free_buf;
86 	}
87 
88 	return 0;
89 
90 err_free_buf:
91 	while (i-- > 0)
92 		free_page((unsigned long)buf[i]);
93 
94 	return -ENOMEM;
95 }
96 
testmgr_free_buf(char * buf[XBUFSIZE])97 static void testmgr_free_buf(char *buf[XBUFSIZE])
98 {
99 	int i;
100 
101 	for (i = 0; i < XBUFSIZE; i++)
102 		free_page((unsigned long)buf[i]);
103 }
104 
sg_init_aead(struct scatterlist * sg,char * xbuf[XBUFSIZE],unsigned int buflen,const void * assoc,unsigned int aad_size)105 static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
106 			 unsigned int buflen, const void *assoc,
107 			 unsigned int aad_size)
108 {
109 	int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
110 	int k, rem;
111 
112 	if (np > XBUFSIZE) {
113 		rem = PAGE_SIZE;
114 		np = XBUFSIZE;
115 	} else {
116 		rem = buflen % PAGE_SIZE;
117 	}
118 
119 	sg_init_table(sg, np + 1);
120 
121 	sg_set_buf(&sg[0], assoc, aad_size);
122 
123 	if (rem)
124 		np--;
125 	for (k = 0; k < np; k++)
126 		sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
127 
128 	if (rem)
129 		sg_set_buf(&sg[k + 1], xbuf[k], rem);
130 }
131 
do_one_aead_op(struct aead_request * req,int ret)132 static inline int do_one_aead_op(struct aead_request *req, int ret)
133 {
134 	struct crypto_wait *wait = req->base.data;
135 
136 	return crypto_wait_req(ret, wait);
137 }
138 
139 struct test_mb_aead_data {
140 	struct scatterlist sg[XBUFSIZE];
141 	struct scatterlist sgout[XBUFSIZE];
142 	struct aead_request *req;
143 	struct crypto_wait wait;
144 	char *xbuf[XBUFSIZE];
145 	char *xoutbuf[XBUFSIZE];
146 	char *axbuf[XBUFSIZE];
147 };
148 
do_mult_aead_op(struct test_mb_aead_data * data,int enc,u32 num_mb,int * rc)149 static int do_mult_aead_op(struct test_mb_aead_data *data, int enc,
150 				u32 num_mb, int *rc)
151 {
152 	int i, err = 0;
153 
154 	/* Fire up a bunch of concurrent requests */
155 	for (i = 0; i < num_mb; i++) {
156 		if (enc == ENCRYPT)
157 			rc[i] = crypto_aead_encrypt(data[i].req);
158 		else
159 			rc[i] = crypto_aead_decrypt(data[i].req);
160 	}
161 
162 	/* Wait for all requests to finish */
163 	for (i = 0; i < num_mb; i++) {
164 		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
165 
166 		if (rc[i]) {
167 			pr_info("concurrent request %d error %d\n", i, rc[i]);
168 			err = rc[i];
169 		}
170 	}
171 
172 	return err;
173 }
174 
test_mb_aead_jiffies(struct test_mb_aead_data * data,int enc,int blen,int secs,u32 num_mb)175 static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc,
176 				int blen, int secs, u32 num_mb)
177 {
178 	unsigned long start, end;
179 	int bcount;
180 	int ret = 0;
181 	int *rc;
182 
183 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
184 	if (!rc)
185 		return -ENOMEM;
186 
187 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
188 	     time_before(jiffies, end); bcount++) {
189 		ret = do_mult_aead_op(data, enc, num_mb, rc);
190 		if (ret)
191 			goto out;
192 	}
193 
194 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
195 		bcount * num_mb, secs, (u64)bcount * blen * num_mb);
196 
197 out:
198 	kfree(rc);
199 	return ret;
200 }
201 
test_mb_aead_cycles(struct test_mb_aead_data * data,int enc,int blen,u32 num_mb)202 static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
203 			       int blen, u32 num_mb)
204 {
205 	unsigned long cycles = 0;
206 	int ret = 0;
207 	int i;
208 	int *rc;
209 
210 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
211 	if (!rc)
212 		return -ENOMEM;
213 
214 	/* Warm-up run. */
215 	for (i = 0; i < 4; i++) {
216 		ret = do_mult_aead_op(data, enc, num_mb, rc);
217 		if (ret)
218 			goto out;
219 	}
220 
221 	/* The real thing. */
222 	for (i = 0; i < 8; i++) {
223 		cycles_t start, end;
224 
225 		start = get_cycles();
226 		ret = do_mult_aead_op(data, enc, num_mb, rc);
227 		end = get_cycles();
228 
229 		if (ret)
230 			goto out;
231 
232 		cycles += end - start;
233 	}
234 
235 	pr_cont("1 operation in %lu cycles (%d bytes)\n",
236 		(cycles + 4) / (8 * num_mb), blen);
237 
238 out:
239 	kfree(rc);
240 	return ret;
241 }
242 
test_mb_aead_speed(const char * algo,int enc,int secs,struct aead_speed_template * template,unsigned int tcount,u8 authsize,unsigned int aad_size,u8 * keysize,u32 num_mb)243 static void test_mb_aead_speed(const char *algo, int enc, int secs,
244 			       struct aead_speed_template *template,
245 			       unsigned int tcount, u8 authsize,
246 			       unsigned int aad_size, u8 *keysize, u32 num_mb)
247 {
248 	struct test_mb_aead_data *data;
249 	struct crypto_aead *tfm;
250 	unsigned int i, j, iv_len;
251 	const int *b_size;
252 	const char *key;
253 	const char *e;
254 	void *assoc;
255 	char *iv;
256 	int ret;
257 
258 
259 	if (aad_size >= PAGE_SIZE) {
260 		pr_err("associate data length (%u) too big\n", aad_size);
261 		return;
262 	}
263 
264 	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
265 	if (!iv)
266 		return;
267 
268 	if (enc == ENCRYPT)
269 		e = "encryption";
270 	else
271 		e = "decryption";
272 
273 	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
274 	if (!data)
275 		goto out_free_iv;
276 
277 	tfm = crypto_alloc_aead(algo, 0, 0);
278 	if (IS_ERR(tfm)) {
279 		pr_err("failed to load transform for %s: %ld\n",
280 			algo, PTR_ERR(tfm));
281 		goto out_free_data;
282 	}
283 
284 	ret = crypto_aead_setauthsize(tfm, authsize);
285 	if (ret) {
286 		pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
287 		       ret);
288 		goto out_free_tfm;
289 	}
290 
291 	for (i = 0; i < num_mb; ++i)
292 		if (testmgr_alloc_buf(data[i].xbuf)) {
293 			while (i--)
294 				testmgr_free_buf(data[i].xbuf);
295 			goto out_free_tfm;
296 		}
297 
298 	for (i = 0; i < num_mb; ++i)
299 		if (testmgr_alloc_buf(data[i].axbuf)) {
300 			while (i--)
301 				testmgr_free_buf(data[i].axbuf);
302 			goto out_free_xbuf;
303 		}
304 
305 	for (i = 0; i < num_mb; ++i)
306 		if (testmgr_alloc_buf(data[i].xoutbuf)) {
307 			while (i--)
308 				testmgr_free_buf(data[i].xoutbuf);
309 			goto out_free_axbuf;
310 		}
311 
312 	for (i = 0; i < num_mb; ++i) {
313 		data[i].req = aead_request_alloc(tfm, GFP_KERNEL);
314 		if (!data[i].req) {
315 			pr_err("alg: aead: Failed to allocate request for %s\n",
316 			       algo);
317 			while (i--)
318 				aead_request_free(data[i].req);
319 			goto out_free_xoutbuf;
320 		}
321 	}
322 
323 	for (i = 0; i < num_mb; ++i) {
324 		crypto_init_wait(&data[i].wait);
325 		aead_request_set_callback(data[i].req,
326 					  CRYPTO_TFM_REQ_MAY_BACKLOG,
327 					  crypto_req_done, &data[i].wait);
328 	}
329 
330 	pr_info("testing speed of multibuffer %s (%s) %s\n", algo,
331 		get_driver_name(crypto_aead, tfm), e);
332 
333 	i = 0;
334 	do {
335 		b_size = aead_sizes;
336 		do {
337 			int bs = round_up(*b_size, crypto_aead_blocksize(tfm));
338 
339 			if (bs + authsize > XBUFSIZE * PAGE_SIZE) {
340 				pr_err("template (%u) too big for buffer (%lu)\n",
341 				       authsize + bs,
342 				       XBUFSIZE * PAGE_SIZE);
343 				goto out;
344 			}
345 
346 			pr_info("test %u (%d bit key, %d byte blocks): ", i,
347 				*keysize * 8, bs);
348 
349 			/* Set up tfm global state, i.e. the key */
350 
351 			memset(tvmem[0], 0xff, PAGE_SIZE);
352 			key = tvmem[0];
353 			for (j = 0; j < tcount; j++) {
354 				if (template[j].klen == *keysize) {
355 					key = template[j].key;
356 					break;
357 				}
358 			}
359 
360 			crypto_aead_clear_flags(tfm, ~0);
361 
362 			ret = crypto_aead_setkey(tfm, key, *keysize);
363 			if (ret) {
364 				pr_err("setkey() failed flags=%x\n",
365 				       crypto_aead_get_flags(tfm));
366 				goto out;
367 			}
368 
369 			iv_len = crypto_aead_ivsize(tfm);
370 			if (iv_len)
371 				memset(iv, 0xff, iv_len);
372 
373 			/* Now setup per request stuff, i.e. buffers */
374 
375 			for (j = 0; j < num_mb; ++j) {
376 				struct test_mb_aead_data *cur = &data[j];
377 
378 				assoc = cur->axbuf[0];
379 				memset(assoc, 0xff, aad_size);
380 
381 				sg_init_aead(cur->sg, cur->xbuf,
382 					     bs + (enc ? 0 : authsize),
383 					     assoc, aad_size);
384 
385 				sg_init_aead(cur->sgout, cur->xoutbuf,
386 					     bs + (enc ? authsize : 0),
387 					     assoc, aad_size);
388 
389 				aead_request_set_ad(cur->req, aad_size);
390 
391 				if (!enc) {
392 
393 					aead_request_set_crypt(cur->req,
394 							       cur->sgout,
395 							       cur->sg,
396 							       bs, iv);
397 					ret = crypto_aead_encrypt(cur->req);
398 					ret = do_one_aead_op(cur->req, ret);
399 
400 					if (ret) {
401 						pr_err("calculating auth failed (%d)\n",
402 						       ret);
403 						break;
404 					}
405 				}
406 
407 				aead_request_set_crypt(cur->req, cur->sg,
408 						       cur->sgout, bs +
409 						       (enc ? 0 : authsize),
410 						       iv);
411 
412 			}
413 
414 			if (secs) {
415 				ret = test_mb_aead_jiffies(data, enc, bs,
416 							   secs, num_mb);
417 				cond_resched();
418 			} else {
419 				ret = test_mb_aead_cycles(data, enc, bs,
420 							  num_mb);
421 			}
422 
423 			if (ret) {
424 				pr_err("%s() failed return code=%d\n", e, ret);
425 				break;
426 			}
427 			b_size++;
428 			i++;
429 		} while (*b_size);
430 		keysize++;
431 	} while (*keysize);
432 
433 out:
434 	for (i = 0; i < num_mb; ++i)
435 		aead_request_free(data[i].req);
436 out_free_xoutbuf:
437 	for (i = 0; i < num_mb; ++i)
438 		testmgr_free_buf(data[i].xoutbuf);
439 out_free_axbuf:
440 	for (i = 0; i < num_mb; ++i)
441 		testmgr_free_buf(data[i].axbuf);
442 out_free_xbuf:
443 	for (i = 0; i < num_mb; ++i)
444 		testmgr_free_buf(data[i].xbuf);
445 out_free_tfm:
446 	crypto_free_aead(tfm);
447 out_free_data:
448 	kfree(data);
449 out_free_iv:
450 	kfree(iv);
451 }
452 
test_aead_jiffies(struct aead_request * req,int enc,int blen,int secs)453 static int test_aead_jiffies(struct aead_request *req, int enc,
454 				int blen, int secs)
455 {
456 	unsigned long start, end;
457 	int bcount;
458 	int ret;
459 
460 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
461 	     time_before(jiffies, end); bcount++) {
462 		if (enc)
463 			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
464 		else
465 			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
466 
467 		if (ret)
468 			return ret;
469 	}
470 
471 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
472 	        bcount, secs, (u64)bcount * blen);
473 	return 0;
474 }
475 
test_aead_cycles(struct aead_request * req,int enc,int blen)476 static int test_aead_cycles(struct aead_request *req, int enc, int blen)
477 {
478 	unsigned long cycles = 0;
479 	int ret = 0;
480 	int i;
481 
482 	/* Warm-up run. */
483 	for (i = 0; i < 4; i++) {
484 		if (enc)
485 			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
486 		else
487 			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
488 
489 		if (ret)
490 			goto out;
491 	}
492 
493 	/* The real thing. */
494 	for (i = 0; i < 8; i++) {
495 		cycles_t start, end;
496 
497 		start = get_cycles();
498 		if (enc)
499 			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
500 		else
501 			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
502 		end = get_cycles();
503 
504 		if (ret)
505 			goto out;
506 
507 		cycles += end - start;
508 	}
509 
510 out:
511 	if (ret == 0)
512 		pr_cont("1 operation in %lu cycles (%d bytes)\n",
513 			(cycles + 4) / 8, blen);
514 
515 	return ret;
516 }
517 
test_aead_speed(const char * algo,int enc,unsigned int secs,struct aead_speed_template * template,unsigned int tcount,u8 authsize,unsigned int aad_size,u8 * keysize)518 static void test_aead_speed(const char *algo, int enc, unsigned int secs,
519 			    struct aead_speed_template *template,
520 			    unsigned int tcount, u8 authsize,
521 			    unsigned int aad_size, u8 *keysize)
522 {
523 	unsigned int i, j;
524 	struct crypto_aead *tfm;
525 	int ret = -ENOMEM;
526 	const char *key;
527 	struct aead_request *req;
528 	struct scatterlist *sg;
529 	struct scatterlist *sgout;
530 	const char *e;
531 	void *assoc;
532 	char *iv;
533 	char *xbuf[XBUFSIZE];
534 	char *xoutbuf[XBUFSIZE];
535 	char *axbuf[XBUFSIZE];
536 	const int *b_size;
537 	unsigned int iv_len;
538 	struct crypto_wait wait;
539 
540 	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
541 	if (!iv)
542 		return;
543 
544 	if (aad_size >= PAGE_SIZE) {
545 		pr_err("associate data length (%u) too big\n", aad_size);
546 		goto out_noxbuf;
547 	}
548 
549 	if (enc == ENCRYPT)
550 		e = "encryption";
551 	else
552 		e = "decryption";
553 
554 	if (testmgr_alloc_buf(xbuf))
555 		goto out_noxbuf;
556 	if (testmgr_alloc_buf(axbuf))
557 		goto out_noaxbuf;
558 	if (testmgr_alloc_buf(xoutbuf))
559 		goto out_nooutbuf;
560 
561 	sg = kmalloc(sizeof(*sg) * 9 * 2, GFP_KERNEL);
562 	if (!sg)
563 		goto out_nosg;
564 	sgout = &sg[9];
565 
566 	tfm = crypto_alloc_aead(algo, 0, 0);
567 	if (IS_ERR(tfm)) {
568 		pr_err("alg: aead: Failed to load transform for %s: %ld\n", algo,
569 		       PTR_ERR(tfm));
570 		goto out_notfm;
571 	}
572 
573 	ret = crypto_aead_setauthsize(tfm, authsize);
574 	if (ret) {
575 		pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
576 		       ret);
577 		goto out_noreq;
578 	}
579 
580 	crypto_init_wait(&wait);
581 	pr_info("testing speed of %s (%s) %s\n", algo,
582 		get_driver_name(crypto_aead, tfm), e);
583 
584 	req = aead_request_alloc(tfm, GFP_KERNEL);
585 	if (!req) {
586 		pr_err("alg: aead: Failed to allocate request for %s\n",
587 		       algo);
588 		goto out_noreq;
589 	}
590 
591 	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
592 				  crypto_req_done, &wait);
593 
594 	i = 0;
595 	do {
596 		b_size = aead_sizes;
597 		do {
598 			u32 bs = round_up(*b_size, crypto_aead_blocksize(tfm));
599 
600 			assoc = axbuf[0];
601 			memset(assoc, 0xff, aad_size);
602 
603 			if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
604 				pr_err("template (%u) too big for tvmem (%lu)\n",
605 				       *keysize + bs,
606 					TVMEMSIZE * PAGE_SIZE);
607 				goto out;
608 			}
609 
610 			key = tvmem[0];
611 			for (j = 0; j < tcount; j++) {
612 				if (template[j].klen == *keysize) {
613 					key = template[j].key;
614 					break;
615 				}
616 			}
617 
618 			ret = crypto_aead_setkey(tfm, key, *keysize);
619 			if (ret) {
620 				pr_err("setkey() failed flags=%x: %d\n",
621 					crypto_aead_get_flags(tfm), ret);
622 				goto out;
623 			}
624 
625 			iv_len = crypto_aead_ivsize(tfm);
626 			if (iv_len)
627 				memset(iv, 0xff, iv_len);
628 
629 			crypto_aead_clear_flags(tfm, ~0);
630 			pr_info("test %u (%d bit key, %d byte blocks): ",
631 				i, *keysize * 8, bs);
632 
633 			memset(tvmem[0], 0xff, PAGE_SIZE);
634 
635 			sg_init_aead(sg, xbuf, bs + (enc ? 0 : authsize),
636 				     assoc, aad_size);
637 
638 			sg_init_aead(sgout, xoutbuf,
639 				     bs + (enc ? authsize : 0), assoc,
640 				     aad_size);
641 
642 			aead_request_set_ad(req, aad_size);
643 
644 			if (!enc) {
645 
646 				/*
647 				 * For decryption we need a proper auth so
648 				 * we do the encryption path once with buffers
649 				 * reversed (input <-> output) to calculate it
650 				 */
651 				aead_request_set_crypt(req, sgout, sg,
652 						       bs, iv);
653 				ret = do_one_aead_op(req,
654 						     crypto_aead_encrypt(req));
655 
656 				if (ret) {
657 					pr_err("calculating auth failed (%d)\n",
658 					       ret);
659 					break;
660 				}
661 			}
662 
663 			aead_request_set_crypt(req, sg, sgout,
664 					       bs + (enc ? 0 : authsize),
665 					       iv);
666 
667 			if (secs) {
668 				ret = test_aead_jiffies(req, enc, bs,
669 							secs);
670 				cond_resched();
671 			} else {
672 				ret = test_aead_cycles(req, enc, bs);
673 			}
674 
675 			if (ret) {
676 				pr_err("%s() failed return code=%d\n", e, ret);
677 				break;
678 			}
679 			b_size++;
680 			i++;
681 		} while (*b_size);
682 		keysize++;
683 	} while (*keysize);
684 
685 out:
686 	aead_request_free(req);
687 out_noreq:
688 	crypto_free_aead(tfm);
689 out_notfm:
690 	kfree(sg);
691 out_nosg:
692 	testmgr_free_buf(xoutbuf);
693 out_nooutbuf:
694 	testmgr_free_buf(axbuf);
695 out_noaxbuf:
696 	testmgr_free_buf(xbuf);
697 out_noxbuf:
698 	kfree(iv);
699 }
700 
test_hash_sg_init(struct scatterlist * sg)701 static void test_hash_sg_init(struct scatterlist *sg)
702 {
703 	int i;
704 
705 	sg_init_table(sg, TVMEMSIZE);
706 	for (i = 0; i < TVMEMSIZE; i++) {
707 		sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
708 		memset(tvmem[i], 0xff, PAGE_SIZE);
709 	}
710 }
711 
do_one_ahash_op(struct ahash_request * req,int ret)712 static inline int do_one_ahash_op(struct ahash_request *req, int ret)
713 {
714 	struct crypto_wait *wait = req->base.data;
715 
716 	return crypto_wait_req(ret, wait);
717 }
718 
test_ahash_jiffies_digest(struct ahash_request * req,int blen,char * out,int secs)719 static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
720 				     char *out, int secs)
721 {
722 	unsigned long start, end;
723 	int bcount;
724 	int ret;
725 
726 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
727 	     time_before(jiffies, end); bcount++) {
728 		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
729 		if (ret)
730 			return ret;
731 	}
732 
733 	pr_cont("%6u opers/sec, %9lu bytes/sec\n",
734 		bcount / secs, ((long)bcount * blen) / secs);
735 
736 	return 0;
737 }
738 
test_ahash_jiffies(struct ahash_request * req,int blen,int plen,char * out,int secs)739 static int test_ahash_jiffies(struct ahash_request *req, int blen,
740 			      int plen, char *out, int secs)
741 {
742 	unsigned long start, end;
743 	int bcount, pcount;
744 	int ret;
745 
746 	if (plen == blen)
747 		return test_ahash_jiffies_digest(req, blen, out, secs);
748 
749 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
750 	     time_before(jiffies, end); bcount++) {
751 		ret = do_one_ahash_op(req, crypto_ahash_init(req));
752 		if (ret)
753 			return ret;
754 		for (pcount = 0; pcount < blen; pcount += plen) {
755 			ret = do_one_ahash_op(req, crypto_ahash_update(req));
756 			if (ret)
757 				return ret;
758 		}
759 		/* we assume there is enough space in 'out' for the result */
760 		ret = do_one_ahash_op(req, crypto_ahash_final(req));
761 		if (ret)
762 			return ret;
763 	}
764 
765 	pr_cont("%6u opers/sec, %9lu bytes/sec\n",
766 		bcount / secs, ((long)bcount * blen) / secs);
767 
768 	return 0;
769 }
770 
test_ahash_cycles_digest(struct ahash_request * req,int blen,char * out)771 static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
772 				    char *out)
773 {
774 	unsigned long cycles = 0;
775 	int ret, i;
776 
777 	/* Warm-up run. */
778 	for (i = 0; i < 4; i++) {
779 		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
780 		if (ret)
781 			goto out;
782 	}
783 
784 	/* The real thing. */
785 	for (i = 0; i < 8; i++) {
786 		cycles_t start, end;
787 
788 		start = get_cycles();
789 
790 		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
791 		if (ret)
792 			goto out;
793 
794 		end = get_cycles();
795 
796 		cycles += end - start;
797 	}
798 
799 out:
800 	if (ret)
801 		return ret;
802 
803 	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
804 		cycles / 8, cycles / (8 * blen));
805 
806 	return 0;
807 }
808 
test_ahash_cycles(struct ahash_request * req,int blen,int plen,char * out)809 static int test_ahash_cycles(struct ahash_request *req, int blen,
810 			     int plen, char *out)
811 {
812 	unsigned long cycles = 0;
813 	int i, pcount, ret;
814 
815 	if (plen == blen)
816 		return test_ahash_cycles_digest(req, blen, out);
817 
818 	/* Warm-up run. */
819 	for (i = 0; i < 4; i++) {
820 		ret = do_one_ahash_op(req, crypto_ahash_init(req));
821 		if (ret)
822 			goto out;
823 		for (pcount = 0; pcount < blen; pcount += plen) {
824 			ret = do_one_ahash_op(req, crypto_ahash_update(req));
825 			if (ret)
826 				goto out;
827 		}
828 		ret = do_one_ahash_op(req, crypto_ahash_final(req));
829 		if (ret)
830 			goto out;
831 	}
832 
833 	/* The real thing. */
834 	for (i = 0; i < 8; i++) {
835 		cycles_t start, end;
836 
837 		start = get_cycles();
838 
839 		ret = do_one_ahash_op(req, crypto_ahash_init(req));
840 		if (ret)
841 			goto out;
842 		for (pcount = 0; pcount < blen; pcount += plen) {
843 			ret = do_one_ahash_op(req, crypto_ahash_update(req));
844 			if (ret)
845 				goto out;
846 		}
847 		ret = do_one_ahash_op(req, crypto_ahash_final(req));
848 		if (ret)
849 			goto out;
850 
851 		end = get_cycles();
852 
853 		cycles += end - start;
854 	}
855 
856 out:
857 	if (ret)
858 		return ret;
859 
860 	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
861 		cycles / 8, cycles / (8 * blen));
862 
863 	return 0;
864 }
865 
test_ahash_speed_common(const char * algo,unsigned int secs,struct hash_speed * speed,unsigned mask)866 static void test_ahash_speed_common(const char *algo, unsigned int secs,
867 				    struct hash_speed *speed, unsigned mask)
868 {
869 	struct scatterlist sg[TVMEMSIZE];
870 	struct crypto_wait wait;
871 	struct ahash_request *req;
872 	struct crypto_ahash *tfm;
873 	char *output;
874 	int i, ret;
875 
876 	tfm = crypto_alloc_ahash(algo, 0, mask);
877 	if (IS_ERR(tfm)) {
878 		pr_err("failed to load transform for %s: %ld\n",
879 		       algo, PTR_ERR(tfm));
880 		return;
881 	}
882 
883 	pr_info("testing speed of async %s (%s)\n", algo,
884 		get_driver_name(crypto_ahash, tfm));
885 
886 	if (crypto_ahash_digestsize(tfm) > MAX_DIGEST_SIZE) {
887 		pr_err("digestsize(%u) > %d\n", crypto_ahash_digestsize(tfm),
888 		       MAX_DIGEST_SIZE);
889 		goto out;
890 	}
891 
892 	test_hash_sg_init(sg);
893 	req = ahash_request_alloc(tfm, GFP_KERNEL);
894 	if (!req) {
895 		pr_err("ahash request allocation failure\n");
896 		goto out;
897 	}
898 
899 	crypto_init_wait(&wait);
900 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
901 				   crypto_req_done, &wait);
902 
903 	output = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL);
904 	if (!output)
905 		goto out_nomem;
906 
907 	for (i = 0; speed[i].blen != 0; i++) {
908 		if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
909 			pr_err("template (%u) too big for tvmem (%lu)\n",
910 			       speed[i].blen, TVMEMSIZE * PAGE_SIZE);
911 			break;
912 		}
913 
914 		if (klen)
915 			crypto_ahash_setkey(tfm, tvmem[0], klen);
916 
917 		pr_info("test%3u "
918 			"(%5u byte blocks,%5u bytes per update,%4u updates): ",
919 			i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
920 
921 		ahash_request_set_crypt(req, sg, output, speed[i].plen);
922 
923 		if (secs) {
924 			ret = test_ahash_jiffies(req, speed[i].blen,
925 						 speed[i].plen, output, secs);
926 			cond_resched();
927 		} else {
928 			ret = test_ahash_cycles(req, speed[i].blen,
929 						speed[i].plen, output);
930 		}
931 
932 		if (ret) {
933 			pr_err("hashing failed ret=%d\n", ret);
934 			break;
935 		}
936 	}
937 
938 	kfree(output);
939 
940 out_nomem:
941 	ahash_request_free(req);
942 
943 out:
944 	crypto_free_ahash(tfm);
945 }
946 
test_ahash_speed(const char * algo,unsigned int secs,struct hash_speed * speed)947 static void test_ahash_speed(const char *algo, unsigned int secs,
948 			     struct hash_speed *speed)
949 {
950 	return test_ahash_speed_common(algo, secs, speed, 0);
951 }
952 
test_hash_speed(const char * algo,unsigned int secs,struct hash_speed * speed)953 static void test_hash_speed(const char *algo, unsigned int secs,
954 			    struct hash_speed *speed)
955 {
956 	return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
957 }
958 
959 struct test_mb_skcipher_data {
960 	struct scatterlist sg[XBUFSIZE];
961 	struct skcipher_request *req;
962 	struct crypto_wait wait;
963 	char *xbuf[XBUFSIZE];
964 };
965 
do_mult_acipher_op(struct test_mb_skcipher_data * data,int enc,u32 num_mb,int * rc)966 static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc,
967 				u32 num_mb, int *rc)
968 {
969 	int i, err = 0;
970 
971 	/* Fire up a bunch of concurrent requests */
972 	for (i = 0; i < num_mb; i++) {
973 		if (enc == ENCRYPT)
974 			rc[i] = crypto_skcipher_encrypt(data[i].req);
975 		else
976 			rc[i] = crypto_skcipher_decrypt(data[i].req);
977 	}
978 
979 	/* Wait for all requests to finish */
980 	for (i = 0; i < num_mb; i++) {
981 		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
982 
983 		if (rc[i]) {
984 			pr_info("concurrent request %d error %d\n", i, rc[i]);
985 			err = rc[i];
986 		}
987 	}
988 
989 	return err;
990 }
991 
test_mb_acipher_jiffies(struct test_mb_skcipher_data * data,int enc,int blen,int secs,u32 num_mb)992 static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc,
993 				int blen, int secs, u32 num_mb)
994 {
995 	unsigned long start, end;
996 	int bcount;
997 	int ret = 0;
998 	int *rc;
999 
1000 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
1001 	if (!rc)
1002 		return -ENOMEM;
1003 
1004 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
1005 	     time_before(jiffies, end); bcount++) {
1006 		ret = do_mult_acipher_op(data, enc, num_mb, rc);
1007 		if (ret)
1008 			goto out;
1009 	}
1010 
1011 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
1012 		bcount * num_mb, secs, (u64)bcount * blen * num_mb);
1013 
1014 out:
1015 	kfree(rc);
1016 	return ret;
1017 }
1018 
test_mb_acipher_cycles(struct test_mb_skcipher_data * data,int enc,int blen,u32 num_mb)1019 static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
1020 			       int blen, u32 num_mb)
1021 {
1022 	unsigned long cycles = 0;
1023 	int ret = 0;
1024 	int i;
1025 	int *rc;
1026 
1027 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
1028 	if (!rc)
1029 		return -ENOMEM;
1030 
1031 	/* Warm-up run. */
1032 	for (i = 0; i < 4; i++) {
1033 		ret = do_mult_acipher_op(data, enc, num_mb, rc);
1034 		if (ret)
1035 			goto out;
1036 	}
1037 
1038 	/* The real thing. */
1039 	for (i = 0; i < 8; i++) {
1040 		cycles_t start, end;
1041 
1042 		start = get_cycles();
1043 		ret = do_mult_acipher_op(data, enc, num_mb, rc);
1044 		end = get_cycles();
1045 
1046 		if (ret)
1047 			goto out;
1048 
1049 		cycles += end - start;
1050 	}
1051 
1052 	pr_cont("1 operation in %lu cycles (%d bytes)\n",
1053 		(cycles + 4) / (8 * num_mb), blen);
1054 
1055 out:
1056 	kfree(rc);
1057 	return ret;
1058 }
1059 
test_mb_skcipher_speed(const char * algo,int enc,int secs,struct cipher_speed_template * template,unsigned int tcount,u8 * keysize,u32 num_mb)1060 static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
1061 				   struct cipher_speed_template *template,
1062 				   unsigned int tcount, u8 *keysize, u32 num_mb)
1063 {
1064 	struct test_mb_skcipher_data *data;
1065 	struct crypto_skcipher *tfm;
1066 	unsigned int i, j, iv_len;
1067 	const int *b_size;
1068 	const char *key;
1069 	const char *e;
1070 	char iv[128];
1071 	int ret;
1072 
1073 	if (enc == ENCRYPT)
1074 		e = "encryption";
1075 	else
1076 		e = "decryption";
1077 
1078 	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
1079 	if (!data)
1080 		return;
1081 
1082 	tfm = crypto_alloc_skcipher(algo, 0, 0);
1083 	if (IS_ERR(tfm)) {
1084 		pr_err("failed to load transform for %s: %ld\n",
1085 			algo, PTR_ERR(tfm));
1086 		goto out_free_data;
1087 	}
1088 
1089 	for (i = 0; i < num_mb; ++i)
1090 		if (testmgr_alloc_buf(data[i].xbuf)) {
1091 			while (i--)
1092 				testmgr_free_buf(data[i].xbuf);
1093 			goto out_free_tfm;
1094 		}
1095 
1096 	for (i = 0; i < num_mb; ++i) {
1097 		data[i].req = skcipher_request_alloc(tfm, GFP_KERNEL);
1098 		if (!data[i].req) {
1099 			pr_err("alg: skcipher: Failed to allocate request for %s\n",
1100 			       algo);
1101 			while (i--)
1102 				skcipher_request_free(data[i].req);
1103 			goto out_free_xbuf;
1104 		}
1105 	}
1106 
1107 	for (i = 0; i < num_mb; ++i) {
1108 		skcipher_request_set_callback(data[i].req,
1109 					      CRYPTO_TFM_REQ_MAY_BACKLOG,
1110 					      crypto_req_done, &data[i].wait);
1111 		crypto_init_wait(&data[i].wait);
1112 	}
1113 
1114 	pr_info("testing speed of multibuffer %s (%s) %s\n", algo,
1115 		get_driver_name(crypto_skcipher, tfm), e);
1116 
1117 	i = 0;
1118 	do {
1119 		b_size = block_sizes;
1120 		do {
1121 			u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
1122 
1123 			if (bs > XBUFSIZE * PAGE_SIZE) {
1124 				pr_err("template (%u) too big for buffer (%lu)\n",
1125 				       bs, XBUFSIZE * PAGE_SIZE);
1126 				goto out;
1127 			}
1128 
1129 			pr_info("test %u (%d bit key, %d byte blocks): ", i,
1130 				*keysize * 8, bs);
1131 
1132 			/* Set up tfm global state, i.e. the key */
1133 
1134 			memset(tvmem[0], 0xff, PAGE_SIZE);
1135 			key = tvmem[0];
1136 			for (j = 0; j < tcount; j++) {
1137 				if (template[j].klen == *keysize) {
1138 					key = template[j].key;
1139 					break;
1140 				}
1141 			}
1142 
1143 			crypto_skcipher_clear_flags(tfm, ~0);
1144 
1145 			ret = crypto_skcipher_setkey(tfm, key, *keysize);
1146 			if (ret) {
1147 				pr_err("setkey() failed flags=%x\n",
1148 				       crypto_skcipher_get_flags(tfm));
1149 				goto out;
1150 			}
1151 
1152 			iv_len = crypto_skcipher_ivsize(tfm);
1153 			if (iv_len)
1154 				memset(&iv, 0xff, iv_len);
1155 
1156 			/* Now setup per request stuff, i.e. buffers */
1157 
1158 			for (j = 0; j < num_mb; ++j) {
1159 				struct test_mb_skcipher_data *cur = &data[j];
1160 				unsigned int k = bs;
1161 				unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE);
1162 				unsigned int p = 0;
1163 
1164 				sg_init_table(cur->sg, pages);
1165 
1166 				while (k > PAGE_SIZE) {
1167 					sg_set_buf(cur->sg + p, cur->xbuf[p],
1168 						   PAGE_SIZE);
1169 					memset(cur->xbuf[p], 0xff, PAGE_SIZE);
1170 					p++;
1171 					k -= PAGE_SIZE;
1172 				}
1173 
1174 				sg_set_buf(cur->sg + p, cur->xbuf[p], k);
1175 				memset(cur->xbuf[p], 0xff, k);
1176 
1177 				skcipher_request_set_crypt(cur->req, cur->sg,
1178 							   cur->sg, bs, iv);
1179 			}
1180 
1181 			if (secs) {
1182 				ret = test_mb_acipher_jiffies(data, enc,
1183 							      bs, secs,
1184 							      num_mb);
1185 				cond_resched();
1186 			} else {
1187 				ret = test_mb_acipher_cycles(data, enc,
1188 							     bs, num_mb);
1189 			}
1190 
1191 			if (ret) {
1192 				pr_err("%s() failed flags=%x\n", e,
1193 				       crypto_skcipher_get_flags(tfm));
1194 				break;
1195 			}
1196 			b_size++;
1197 			i++;
1198 		} while (*b_size);
1199 		keysize++;
1200 	} while (*keysize);
1201 
1202 out:
1203 	for (i = 0; i < num_mb; ++i)
1204 		skcipher_request_free(data[i].req);
1205 out_free_xbuf:
1206 	for (i = 0; i < num_mb; ++i)
1207 		testmgr_free_buf(data[i].xbuf);
1208 out_free_tfm:
1209 	crypto_free_skcipher(tfm);
1210 out_free_data:
1211 	kfree(data);
1212 }
1213 
do_one_acipher_op(struct skcipher_request * req,int ret)1214 static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
1215 {
1216 	struct crypto_wait *wait = req->base.data;
1217 
1218 	return crypto_wait_req(ret, wait);
1219 }
1220 
test_acipher_jiffies(struct skcipher_request * req,int enc,int blen,int secs)1221 static int test_acipher_jiffies(struct skcipher_request *req, int enc,
1222 				int blen, int secs)
1223 {
1224 	unsigned long start, end;
1225 	int bcount;
1226 	int ret;
1227 
1228 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
1229 	     time_before(jiffies, end); bcount++) {
1230 		if (enc)
1231 			ret = do_one_acipher_op(req,
1232 						crypto_skcipher_encrypt(req));
1233 		else
1234 			ret = do_one_acipher_op(req,
1235 						crypto_skcipher_decrypt(req));
1236 
1237 		if (ret)
1238 			return ret;
1239 	}
1240 
1241 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
1242 		bcount, secs, (u64)bcount * blen);
1243 	return 0;
1244 }
1245 
test_acipher_cycles(struct skcipher_request * req,int enc,int blen)1246 static int test_acipher_cycles(struct skcipher_request *req, int enc,
1247 			       int blen)
1248 {
1249 	unsigned long cycles = 0;
1250 	int ret = 0;
1251 	int i;
1252 
1253 	/* Warm-up run. */
1254 	for (i = 0; i < 4; i++) {
1255 		if (enc)
1256 			ret = do_one_acipher_op(req,
1257 						crypto_skcipher_encrypt(req));
1258 		else
1259 			ret = do_one_acipher_op(req,
1260 						crypto_skcipher_decrypt(req));
1261 
1262 		if (ret)
1263 			goto out;
1264 	}
1265 
1266 	/* The real thing. */
1267 	for (i = 0; i < 8; i++) {
1268 		cycles_t start, end;
1269 
1270 		start = get_cycles();
1271 		if (enc)
1272 			ret = do_one_acipher_op(req,
1273 						crypto_skcipher_encrypt(req));
1274 		else
1275 			ret = do_one_acipher_op(req,
1276 						crypto_skcipher_decrypt(req));
1277 		end = get_cycles();
1278 
1279 		if (ret)
1280 			goto out;
1281 
1282 		cycles += end - start;
1283 	}
1284 
1285 out:
1286 	if (ret == 0)
1287 		pr_cont("1 operation in %lu cycles (%d bytes)\n",
1288 			(cycles + 4) / 8, blen);
1289 
1290 	return ret;
1291 }
1292 
test_skcipher_speed(const char * algo,int enc,unsigned int secs,struct cipher_speed_template * template,unsigned int tcount,u8 * keysize,bool async)1293 static void test_skcipher_speed(const char *algo, int enc, unsigned int secs,
1294 				struct cipher_speed_template *template,
1295 				unsigned int tcount, u8 *keysize, bool async)
1296 {
1297 	unsigned int ret, i, j, k, iv_len;
1298 	struct crypto_wait wait;
1299 	const char *key;
1300 	char iv[128];
1301 	struct skcipher_request *req;
1302 	struct crypto_skcipher *tfm;
1303 	const int *b_size;
1304 	const char *e;
1305 
1306 	if (enc == ENCRYPT)
1307 		e = "encryption";
1308 	else
1309 		e = "decryption";
1310 
1311 	crypto_init_wait(&wait);
1312 
1313 	tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC);
1314 
1315 	if (IS_ERR(tfm)) {
1316 		pr_err("failed to load transform for %s: %ld\n", algo,
1317 		       PTR_ERR(tfm));
1318 		return;
1319 	}
1320 
1321 	pr_info("testing speed of %s %s (%s) %s\n", async ? "async" : "sync",
1322 		algo, get_driver_name(crypto_skcipher, tfm), e);
1323 
1324 	req = skcipher_request_alloc(tfm, GFP_KERNEL);
1325 	if (!req) {
1326 		pr_err("skcipher: Failed to allocate request for %s\n", algo);
1327 		goto out;
1328 	}
1329 
1330 	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1331 				      crypto_req_done, &wait);
1332 
1333 	i = 0;
1334 	do {
1335 		b_size = block_sizes;
1336 
1337 		do {
1338 			u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
1339 			struct scatterlist sg[TVMEMSIZE];
1340 
1341 			if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
1342 				pr_err("template (%u) too big for "
1343 				       "tvmem (%lu)\n", *keysize + bs,
1344 				       TVMEMSIZE * PAGE_SIZE);
1345 				goto out_free_req;
1346 			}
1347 
1348 			pr_info("test %u (%d bit key, %d byte blocks): ", i,
1349 				*keysize * 8, bs);
1350 
1351 			memset(tvmem[0], 0xff, PAGE_SIZE);
1352 
1353 			/* set key, plain text and IV */
1354 			key = tvmem[0];
1355 			for (j = 0; j < tcount; j++) {
1356 				if (template[j].klen == *keysize) {
1357 					key = template[j].key;
1358 					break;
1359 				}
1360 			}
1361 
1362 			crypto_skcipher_clear_flags(tfm, ~0);
1363 
1364 			ret = crypto_skcipher_setkey(tfm, key, *keysize);
1365 			if (ret) {
1366 				pr_err("setkey() failed flags=%x\n",
1367 					crypto_skcipher_get_flags(tfm));
1368 				goto out_free_req;
1369 			}
1370 
1371 			k = *keysize + bs;
1372 			sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE));
1373 
1374 			if (k > PAGE_SIZE) {
1375 				sg_set_buf(sg, tvmem[0] + *keysize,
1376 				   PAGE_SIZE - *keysize);
1377 				k -= PAGE_SIZE;
1378 				j = 1;
1379 				while (k > PAGE_SIZE) {
1380 					sg_set_buf(sg + j, tvmem[j], PAGE_SIZE);
1381 					memset(tvmem[j], 0xff, PAGE_SIZE);
1382 					j++;
1383 					k -= PAGE_SIZE;
1384 				}
1385 				sg_set_buf(sg + j, tvmem[j], k);
1386 				memset(tvmem[j], 0xff, k);
1387 			} else {
1388 				sg_set_buf(sg, tvmem[0] + *keysize, bs);
1389 			}
1390 
1391 			iv_len = crypto_skcipher_ivsize(tfm);
1392 			if (iv_len)
1393 				memset(&iv, 0xff, iv_len);
1394 
1395 			skcipher_request_set_crypt(req, sg, sg, bs, iv);
1396 
1397 			if (secs) {
1398 				ret = test_acipher_jiffies(req, enc,
1399 							   bs, secs);
1400 				cond_resched();
1401 			} else {
1402 				ret = test_acipher_cycles(req, enc,
1403 							  bs);
1404 			}
1405 
1406 			if (ret) {
1407 				pr_err("%s() failed flags=%x\n", e,
1408 				       crypto_skcipher_get_flags(tfm));
1409 				break;
1410 			}
1411 			b_size++;
1412 			i++;
1413 		} while (*b_size);
1414 		keysize++;
1415 	} while (*keysize);
1416 
1417 out_free_req:
1418 	skcipher_request_free(req);
1419 out:
1420 	crypto_free_skcipher(tfm);
1421 }
1422 
test_acipher_speed(const char * algo,int enc,unsigned int secs,struct cipher_speed_template * template,unsigned int tcount,u8 * keysize)1423 static void test_acipher_speed(const char *algo, int enc, unsigned int secs,
1424 			       struct cipher_speed_template *template,
1425 			       unsigned int tcount, u8 *keysize)
1426 {
1427 	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
1428 				   true);
1429 }
1430 
test_cipher_speed(const char * algo,int enc,unsigned int secs,struct cipher_speed_template * template,unsigned int tcount,u8 * keysize)1431 static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
1432 			      struct cipher_speed_template *template,
1433 			      unsigned int tcount, u8 *keysize)
1434 {
1435 	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
1436 				   false);
1437 }
1438 
tcrypt_test(const char * alg)1439 static inline int tcrypt_test(const char *alg)
1440 {
1441 	int ret;
1442 
1443 	pr_debug("testing %s\n", alg);
1444 
1445 	ret = alg_test(alg, alg, 0, 0);
1446 	/* non-fips algs return -EINVAL or -ECANCELED in fips mode */
1447 	if (fips_enabled && (ret == -EINVAL || ret == -ECANCELED))
1448 		ret = 0;
1449 	return ret;
1450 }
1451 
do_test(const char * alg,u32 type,u32 mask,int m,u32 num_mb)1452 static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
1453 {
1454 	int i;
1455 	int ret = 0;
1456 
1457 	switch (m) {
1458 	case 0:
1459 		if (alg) {
1460 			if (!crypto_has_alg(alg, type,
1461 					    mask ?: CRYPTO_ALG_TYPE_MASK))
1462 				ret = -ENOENT;
1463 			break;
1464 		}
1465 
1466 		for (i = 1; i < 200; i++)
1467 			ret = min(ret, do_test(NULL, 0, 0, i, num_mb));
1468 		break;
1469 
1470 	case 1:
1471 		ret = min(ret, tcrypt_test("md5"));
1472 		break;
1473 
1474 	case 2:
1475 		ret = min(ret, tcrypt_test("sha1"));
1476 		break;
1477 
1478 	case 3:
1479 		ret = min(ret, tcrypt_test("ecb(des)"));
1480 		ret = min(ret, tcrypt_test("cbc(des)"));
1481 		ret = min(ret, tcrypt_test("ctr(des)"));
1482 		break;
1483 
1484 	case 4:
1485 		ret = min(ret, tcrypt_test("ecb(des3_ede)"));
1486 		ret = min(ret, tcrypt_test("cbc(des3_ede)"));
1487 		ret = min(ret, tcrypt_test("ctr(des3_ede)"));
1488 		break;
1489 
1490 	case 5:
1491 		ret = min(ret, tcrypt_test("md4"));
1492 		break;
1493 
1494 	case 6:
1495 		ret = min(ret, tcrypt_test("sha256"));
1496 		break;
1497 
1498 	case 7:
1499 		ret = min(ret, tcrypt_test("ecb(blowfish)"));
1500 		ret = min(ret, tcrypt_test("cbc(blowfish)"));
1501 		ret = min(ret, tcrypt_test("ctr(blowfish)"));
1502 		break;
1503 
1504 	case 8:
1505 		ret = min(ret, tcrypt_test("ecb(twofish)"));
1506 		ret = min(ret, tcrypt_test("cbc(twofish)"));
1507 		ret = min(ret, tcrypt_test("ctr(twofish)"));
1508 		ret = min(ret, tcrypt_test("lrw(twofish)"));
1509 		ret = min(ret, tcrypt_test("xts(twofish)"));
1510 		break;
1511 
1512 	case 9:
1513 		ret = min(ret, tcrypt_test("ecb(serpent)"));
1514 		ret = min(ret, tcrypt_test("cbc(serpent)"));
1515 		ret = min(ret, tcrypt_test("ctr(serpent)"));
1516 		ret = min(ret, tcrypt_test("lrw(serpent)"));
1517 		ret = min(ret, tcrypt_test("xts(serpent)"));
1518 		break;
1519 
1520 	case 10:
1521 		ret = min(ret, tcrypt_test("ecb(aes)"));
1522 		ret = min(ret, tcrypt_test("cbc(aes)"));
1523 		ret = min(ret, tcrypt_test("lrw(aes)"));
1524 		ret = min(ret, tcrypt_test("xts(aes)"));
1525 		ret = min(ret, tcrypt_test("ctr(aes)"));
1526 		ret = min(ret, tcrypt_test("rfc3686(ctr(aes))"));
1527 		ret = min(ret, tcrypt_test("xctr(aes)"));
1528 		break;
1529 
1530 	case 11:
1531 		ret = min(ret, tcrypt_test("sha384"));
1532 		break;
1533 
1534 	case 12:
1535 		ret = min(ret, tcrypt_test("sha512"));
1536 		break;
1537 
1538 	case 13:
1539 		ret = min(ret, tcrypt_test("deflate"));
1540 		break;
1541 
1542 	case 14:
1543 		ret = min(ret, tcrypt_test("ecb(cast5)"));
1544 		ret = min(ret, tcrypt_test("cbc(cast5)"));
1545 		ret = min(ret, tcrypt_test("ctr(cast5)"));
1546 		break;
1547 
1548 	case 15:
1549 		ret = min(ret, tcrypt_test("ecb(cast6)"));
1550 		ret = min(ret, tcrypt_test("cbc(cast6)"));
1551 		ret = min(ret, tcrypt_test("ctr(cast6)"));
1552 		ret = min(ret, tcrypt_test("lrw(cast6)"));
1553 		ret = min(ret, tcrypt_test("xts(cast6)"));
1554 		break;
1555 
1556 	case 16:
1557 		ret = min(ret, tcrypt_test("ecb(arc4)"));
1558 		break;
1559 
1560 	case 17:
1561 		ret = min(ret, tcrypt_test("michael_mic"));
1562 		break;
1563 
1564 	case 18:
1565 		ret = min(ret, tcrypt_test("crc32c"));
1566 		break;
1567 
1568 	case 19:
1569 		ret = min(ret, tcrypt_test("ecb(tea)"));
1570 		break;
1571 
1572 	case 20:
1573 		ret = min(ret, tcrypt_test("ecb(xtea)"));
1574 		break;
1575 
1576 	case 21:
1577 		ret = min(ret, tcrypt_test("ecb(khazad)"));
1578 		break;
1579 
1580 	case 22:
1581 		ret = min(ret, tcrypt_test("wp512"));
1582 		break;
1583 
1584 	case 23:
1585 		ret = min(ret, tcrypt_test("wp384"));
1586 		break;
1587 
1588 	case 24:
1589 		ret = min(ret, tcrypt_test("wp256"));
1590 		break;
1591 
1592 	case 26:
1593 		ret = min(ret, tcrypt_test("ecb(anubis)"));
1594 		ret = min(ret, tcrypt_test("cbc(anubis)"));
1595 		break;
1596 
1597 	case 30:
1598 		ret = min(ret, tcrypt_test("ecb(xeta)"));
1599 		break;
1600 
1601 	case 31:
1602 		ret = min(ret, tcrypt_test("pcbc(fcrypt)"));
1603 		break;
1604 
1605 	case 32:
1606 		ret = min(ret, tcrypt_test("ecb(camellia)"));
1607 		ret = min(ret, tcrypt_test("cbc(camellia)"));
1608 		ret = min(ret, tcrypt_test("ctr(camellia)"));
1609 		ret = min(ret, tcrypt_test("lrw(camellia)"));
1610 		ret = min(ret, tcrypt_test("xts(camellia)"));
1611 		break;
1612 
1613 	case 33:
1614 		ret = min(ret, tcrypt_test("sha224"));
1615 		break;
1616 
1617 	case 35:
1618 		ret = min(ret, tcrypt_test("gcm(aes)"));
1619 		break;
1620 
1621 	case 36:
1622 		ret = min(ret, tcrypt_test("lzo"));
1623 		break;
1624 
1625 	case 37:
1626 		ret = min(ret, tcrypt_test("ccm(aes)"));
1627 		break;
1628 
1629 	case 38:
1630 		ret = min(ret, tcrypt_test("cts(cbc(aes))"));
1631 		break;
1632 
1633         case 39:
1634 		ret = min(ret, tcrypt_test("xxhash64"));
1635 		break;
1636 
1637         case 40:
1638 		ret = min(ret, tcrypt_test("rmd160"));
1639 		break;
1640 
1641 	case 42:
1642 		ret = min(ret, tcrypt_test("blake2b-512"));
1643 		break;
1644 
1645 	case 43:
1646 		ret = min(ret, tcrypt_test("ecb(seed)"));
1647 		break;
1648 
1649 	case 45:
1650 		ret = min(ret, tcrypt_test("rfc4309(ccm(aes))"));
1651 		break;
1652 
1653 	case 46:
1654 		ret = min(ret, tcrypt_test("ghash"));
1655 		break;
1656 
1657 	case 47:
1658 		ret = min(ret, tcrypt_test("crct10dif"));
1659 		break;
1660 
1661 	case 48:
1662 		ret = min(ret, tcrypt_test("sha3-224"));
1663 		break;
1664 
1665 	case 49:
1666 		ret = min(ret, tcrypt_test("sha3-256"));
1667 		break;
1668 
1669 	case 50:
1670 		ret = min(ret, tcrypt_test("sha3-384"));
1671 		break;
1672 
1673 	case 51:
1674 		ret = min(ret, tcrypt_test("sha3-512"));
1675 		break;
1676 
1677 	case 52:
1678 		ret = min(ret, tcrypt_test("sm3"));
1679 		break;
1680 
1681 	case 53:
1682 		ret = min(ret, tcrypt_test("streebog256"));
1683 		break;
1684 
1685 	case 54:
1686 		ret = min(ret, tcrypt_test("streebog512"));
1687 		break;
1688 
1689 	case 55:
1690 		ret = min(ret, tcrypt_test("gcm(sm4)"));
1691 		break;
1692 
1693 	case 56:
1694 		ret = min(ret, tcrypt_test("ccm(sm4)"));
1695 		break;
1696 
1697 	case 57:
1698 		ret = min(ret, tcrypt_test("polyval"));
1699 		break;
1700 
1701 	case 58:
1702 		ret = min(ret, tcrypt_test("gcm(aria)"));
1703 		break;
1704 
1705 	case 59:
1706 		ret = min(ret, tcrypt_test("cts(cbc(sm4))"));
1707 		break;
1708 
1709 	case 100:
1710 		ret = min(ret, tcrypt_test("hmac(md5)"));
1711 		break;
1712 
1713 	case 101:
1714 		ret = min(ret, tcrypt_test("hmac(sha1)"));
1715 		break;
1716 
1717 	case 102:
1718 		ret = min(ret, tcrypt_test("hmac(sha256)"));
1719 		break;
1720 
1721 	case 103:
1722 		ret = min(ret, tcrypt_test("hmac(sha384)"));
1723 		break;
1724 
1725 	case 104:
1726 		ret = min(ret, tcrypt_test("hmac(sha512)"));
1727 		break;
1728 
1729 	case 105:
1730 		ret = min(ret, tcrypt_test("hmac(sha224)"));
1731 		break;
1732 
1733 	case 106:
1734 		ret = min(ret, tcrypt_test("xcbc(aes)"));
1735 		break;
1736 
1737 	case 108:
1738 		ret = min(ret, tcrypt_test("hmac(rmd160)"));
1739 		break;
1740 
1741 	case 109:
1742 		ret = min(ret, tcrypt_test("vmac64(aes)"));
1743 		break;
1744 
1745 	case 111:
1746 		ret = min(ret, tcrypt_test("hmac(sha3-224)"));
1747 		break;
1748 
1749 	case 112:
1750 		ret = min(ret, tcrypt_test("hmac(sha3-256)"));
1751 		break;
1752 
1753 	case 113:
1754 		ret = min(ret, tcrypt_test("hmac(sha3-384)"));
1755 		break;
1756 
1757 	case 114:
1758 		ret = min(ret, tcrypt_test("hmac(sha3-512)"));
1759 		break;
1760 
1761 	case 115:
1762 		ret = min(ret, tcrypt_test("hmac(streebog256)"));
1763 		break;
1764 
1765 	case 116:
1766 		ret = min(ret, tcrypt_test("hmac(streebog512)"));
1767 		break;
1768 
1769 	case 150:
1770 		ret = min(ret, tcrypt_test("ansi_cprng"));
1771 		break;
1772 
1773 	case 151:
1774 		ret = min(ret, tcrypt_test("rfc4106(gcm(aes))"));
1775 		break;
1776 
1777 	case 152:
1778 		ret = min(ret, tcrypt_test("rfc4543(gcm(aes))"));
1779 		break;
1780 
1781 	case 153:
1782 		ret = min(ret, tcrypt_test("cmac(aes)"));
1783 		break;
1784 
1785 	case 154:
1786 		ret = min(ret, tcrypt_test("cmac(des3_ede)"));
1787 		break;
1788 
1789 	case 155:
1790 		ret = min(ret, tcrypt_test("authenc(hmac(sha1),cbc(aes))"));
1791 		break;
1792 
1793 	case 156:
1794 		ret = min(ret, tcrypt_test("authenc(hmac(md5),ecb(cipher_null))"));
1795 		break;
1796 
1797 	case 157:
1798 		ret = min(ret, tcrypt_test("authenc(hmac(sha1),ecb(cipher_null))"));
1799 		break;
1800 
1801 	case 158:
1802 		ret = min(ret, tcrypt_test("cbcmac(sm4)"));
1803 		break;
1804 
1805 	case 159:
1806 		ret = min(ret, tcrypt_test("cmac(sm4)"));
1807 		break;
1808 
1809 	case 160:
1810 		ret = min(ret, tcrypt_test("xcbc(sm4)"));
1811 		break;
1812 
1813 	case 181:
1814 		ret = min(ret, tcrypt_test("authenc(hmac(sha1),cbc(des))"));
1815 		break;
1816 	case 182:
1817 		ret = min(ret, tcrypt_test("authenc(hmac(sha1),cbc(des3_ede))"));
1818 		break;
1819 	case 183:
1820 		ret = min(ret, tcrypt_test("authenc(hmac(sha224),cbc(des))"));
1821 		break;
1822 	case 184:
1823 		ret = min(ret, tcrypt_test("authenc(hmac(sha224),cbc(des3_ede))"));
1824 		break;
1825 	case 185:
1826 		ret = min(ret, tcrypt_test("authenc(hmac(sha256),cbc(des))"));
1827 		break;
1828 	case 186:
1829 		ret = min(ret, tcrypt_test("authenc(hmac(sha256),cbc(des3_ede))"));
1830 		break;
1831 	case 187:
1832 		ret = min(ret, tcrypt_test("authenc(hmac(sha384),cbc(des))"));
1833 		break;
1834 	case 188:
1835 		ret = min(ret, tcrypt_test("authenc(hmac(sha384),cbc(des3_ede))"));
1836 		break;
1837 	case 189:
1838 		ret = min(ret, tcrypt_test("authenc(hmac(sha512),cbc(des))"));
1839 		break;
1840 	case 190:
1841 		ret = min(ret, tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))"));
1842 		break;
1843 	case 191:
1844 		ret = min(ret, tcrypt_test("ecb(sm4)"));
1845 		ret = min(ret, tcrypt_test("cbc(sm4)"));
1846 		ret = min(ret, tcrypt_test("ctr(sm4)"));
1847 		ret = min(ret, tcrypt_test("xts(sm4)"));
1848 		break;
1849 	case 192:
1850 		ret = min(ret, tcrypt_test("ecb(aria)"));
1851 		ret = min(ret, tcrypt_test("cbc(aria)"));
1852 		ret = min(ret, tcrypt_test("ctr(aria)"));
1853 		break;
1854 	case 193:
1855 		ret = min(ret, tcrypt_test("ffdhe2048(dh)"));
1856 		break;
1857 	case 200:
1858 		test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
1859 				speed_template_16_24_32);
1860 		test_cipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
1861 				speed_template_16_24_32);
1862 		test_cipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
1863 				speed_template_16_24_32);
1864 		test_cipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
1865 				speed_template_16_24_32);
1866 		test_cipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
1867 				speed_template_32_40_48);
1868 		test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
1869 				speed_template_32_40_48);
1870 		test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
1871 				speed_template_32_64);
1872 		test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
1873 				speed_template_32_64);
1874 		test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
1875 				speed_template_16_24_32);
1876 		test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
1877 				speed_template_16_24_32);
1878 		test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
1879 				speed_template_16_24_32);
1880 		test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
1881 				speed_template_16_24_32);
1882 		break;
1883 
1884 	case 201:
1885 		test_cipher_speed("ecb(des3_ede)", ENCRYPT, sec,
1886 				des3_speed_template, DES3_SPEED_VECTORS,
1887 				speed_template_24);
1888 		test_cipher_speed("ecb(des3_ede)", DECRYPT, sec,
1889 				des3_speed_template, DES3_SPEED_VECTORS,
1890 				speed_template_24);
1891 		test_cipher_speed("cbc(des3_ede)", ENCRYPT, sec,
1892 				des3_speed_template, DES3_SPEED_VECTORS,
1893 				speed_template_24);
1894 		test_cipher_speed("cbc(des3_ede)", DECRYPT, sec,
1895 				des3_speed_template, DES3_SPEED_VECTORS,
1896 				speed_template_24);
1897 		test_cipher_speed("ctr(des3_ede)", ENCRYPT, sec,
1898 				des3_speed_template, DES3_SPEED_VECTORS,
1899 				speed_template_24);
1900 		test_cipher_speed("ctr(des3_ede)", DECRYPT, sec,
1901 				des3_speed_template, DES3_SPEED_VECTORS,
1902 				speed_template_24);
1903 		break;
1904 
1905 	case 202:
1906 		test_cipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
1907 				speed_template_16_24_32);
1908 		test_cipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
1909 				speed_template_16_24_32);
1910 		test_cipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
1911 				speed_template_16_24_32);
1912 		test_cipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
1913 				speed_template_16_24_32);
1914 		test_cipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
1915 				speed_template_16_24_32);
1916 		test_cipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
1917 				speed_template_16_24_32);
1918 		test_cipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
1919 				speed_template_32_40_48);
1920 		test_cipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
1921 				speed_template_32_40_48);
1922 		test_cipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
1923 				speed_template_32_48_64);
1924 		test_cipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
1925 				speed_template_32_48_64);
1926 		break;
1927 
1928 	case 203:
1929 		test_cipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
1930 				  speed_template_8_32);
1931 		test_cipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
1932 				  speed_template_8_32);
1933 		test_cipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
1934 				  speed_template_8_32);
1935 		test_cipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
1936 				  speed_template_8_32);
1937 		test_cipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
1938 				  speed_template_8_32);
1939 		test_cipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
1940 				  speed_template_8_32);
1941 		break;
1942 
1943 	case 204:
1944 		test_cipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
1945 				  speed_template_8);
1946 		test_cipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
1947 				  speed_template_8);
1948 		test_cipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
1949 				  speed_template_8);
1950 		test_cipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
1951 				  speed_template_8);
1952 		break;
1953 
1954 	case 205:
1955 		test_cipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
1956 				speed_template_16_24_32);
1957 		test_cipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
1958 				speed_template_16_24_32);
1959 		test_cipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
1960 				speed_template_16_24_32);
1961 		test_cipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
1962 				speed_template_16_24_32);
1963 		test_cipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
1964 				speed_template_16_24_32);
1965 		test_cipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
1966 				speed_template_16_24_32);
1967 		test_cipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
1968 				speed_template_32_40_48);
1969 		test_cipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
1970 				speed_template_32_40_48);
1971 		test_cipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
1972 				speed_template_32_48_64);
1973 		test_cipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
1974 				speed_template_32_48_64);
1975 		break;
1976 
1977 	case 207:
1978 		test_cipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
1979 				  speed_template_16_32);
1980 		test_cipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
1981 				  speed_template_16_32);
1982 		test_cipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
1983 				  speed_template_16_32);
1984 		test_cipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
1985 				  speed_template_16_32);
1986 		test_cipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
1987 				  speed_template_16_32);
1988 		test_cipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
1989 				  speed_template_16_32);
1990 		test_cipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
1991 				  speed_template_32_48);
1992 		test_cipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
1993 				  speed_template_32_48);
1994 		test_cipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
1995 				  speed_template_32_64);
1996 		test_cipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
1997 				  speed_template_32_64);
1998 		break;
1999 
2000 	case 208:
2001 		test_cipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2002 				  speed_template_8);
2003 		break;
2004 
2005 	case 209:
2006 		test_cipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2007 				  speed_template_8_16);
2008 		test_cipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2009 				  speed_template_8_16);
2010 		test_cipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2011 				  speed_template_8_16);
2012 		test_cipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2013 				  speed_template_8_16);
2014 		test_cipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2015 				  speed_template_8_16);
2016 		test_cipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2017 				  speed_template_8_16);
2018 		break;
2019 
2020 	case 210:
2021 		test_cipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2022 				  speed_template_16_32);
2023 		test_cipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2024 				  speed_template_16_32);
2025 		test_cipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2026 				  speed_template_16_32);
2027 		test_cipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2028 				  speed_template_16_32);
2029 		test_cipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2030 				  speed_template_16_32);
2031 		test_cipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2032 				  speed_template_16_32);
2033 		test_cipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2034 				  speed_template_32_48);
2035 		test_cipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2036 				  speed_template_32_48);
2037 		test_cipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2038 				  speed_template_32_64);
2039 		test_cipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2040 				  speed_template_32_64);
2041 		break;
2042 
2043 	case 211:
2044 		test_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec,
2045 				NULL, 0, 16, 16, aead_speed_template_20_28_36);
2046 		test_aead_speed("gcm(aes)", ENCRYPT, sec,
2047 				NULL, 0, 16, 8, speed_template_16_24_32);
2048 		test_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec,
2049 				NULL, 0, 16, 16, aead_speed_template_20_28_36);
2050 		test_aead_speed("gcm(aes)", DECRYPT, sec,
2051 				NULL, 0, 16, 8, speed_template_16_24_32);
2052 		break;
2053 
2054 	case 212:
2055 		test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec,
2056 				NULL, 0, 16, 16, aead_speed_template_19);
2057 		test_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec,
2058 				NULL, 0, 16, 16, aead_speed_template_19);
2059 		break;
2060 
2061 	case 213:
2062 		test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec,
2063 				NULL, 0, 16, 8, aead_speed_template_36);
2064 		test_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, sec,
2065 				NULL, 0, 16, 8, aead_speed_template_36);
2066 		break;
2067 
2068 	case 214:
2069 		test_cipher_speed("chacha20", ENCRYPT, sec, NULL, 0,
2070 				  speed_template_32);
2071 		break;
2072 
2073 	case 215:
2074 		test_mb_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, NULL,
2075 				   0, 16, 16, aead_speed_template_20_28_36, num_mb);
2076 		test_mb_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8,
2077 				   speed_template_16_24_32, num_mb);
2078 		test_mb_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, NULL,
2079 				   0, 16, 16, aead_speed_template_20_28_36, num_mb);
2080 		test_mb_aead_speed("gcm(aes)", DECRYPT, sec, NULL, 0, 16, 8,
2081 				   speed_template_16_24_32, num_mb);
2082 		break;
2083 
2084 	case 216:
2085 		test_mb_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0,
2086 				   16, 16, aead_speed_template_19, num_mb);
2087 		test_mb_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, NULL, 0,
2088 				   16, 16, aead_speed_template_19, num_mb);
2089 		break;
2090 
2091 	case 217:
2092 		test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT,
2093 				   sec, NULL, 0, 16, 8, aead_speed_template_36,
2094 				   num_mb);
2095 		test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT,
2096 				   sec, NULL, 0, 16, 8, aead_speed_template_36,
2097 				   num_mb);
2098 		break;
2099 
2100 	case 218:
2101 		test_cipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
2102 				speed_template_16);
2103 		test_cipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
2104 				speed_template_16);
2105 		test_cipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
2106 				speed_template_16);
2107 		test_cipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
2108 				speed_template_16);
2109 		test_cipher_speed("cts(cbc(sm4))", ENCRYPT, sec, NULL, 0,
2110 				speed_template_16);
2111 		test_cipher_speed("cts(cbc(sm4))", DECRYPT, sec, NULL, 0,
2112 				speed_template_16);
2113 		test_cipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
2114 				speed_template_16);
2115 		test_cipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
2116 				speed_template_16);
2117 		test_cipher_speed("xts(sm4)", ENCRYPT, sec, NULL, 0,
2118 				speed_template_32);
2119 		test_cipher_speed("xts(sm4)", DECRYPT, sec, NULL, 0,
2120 				speed_template_32);
2121 		break;
2122 
2123 	case 219:
2124 		test_cipher_speed("adiantum(xchacha12,aes)", ENCRYPT, sec, NULL,
2125 				  0, speed_template_32);
2126 		test_cipher_speed("adiantum(xchacha12,aes)", DECRYPT, sec, NULL,
2127 				  0, speed_template_32);
2128 		test_cipher_speed("adiantum(xchacha20,aes)", ENCRYPT, sec, NULL,
2129 				  0, speed_template_32);
2130 		test_cipher_speed("adiantum(xchacha20,aes)", DECRYPT, sec, NULL,
2131 				  0, speed_template_32);
2132 		break;
2133 
2134 	case 220:
2135 		test_acipher_speed("essiv(cbc(aes),sha256)",
2136 				  ENCRYPT, sec, NULL, 0,
2137 				  speed_template_16_24_32);
2138 		test_acipher_speed("essiv(cbc(aes),sha256)",
2139 				  DECRYPT, sec, NULL, 0,
2140 				  speed_template_16_24_32);
2141 		break;
2142 
2143 	case 221:
2144 		test_aead_speed("aegis128", ENCRYPT, sec,
2145 				NULL, 0, 16, 8, speed_template_16);
2146 		test_aead_speed("aegis128", DECRYPT, sec,
2147 				NULL, 0, 16, 8, speed_template_16);
2148 		break;
2149 
2150 	case 222:
2151 		test_aead_speed("gcm(sm4)", ENCRYPT, sec,
2152 				NULL, 0, 16, 8, speed_template_16);
2153 		test_aead_speed("gcm(sm4)", DECRYPT, sec,
2154 				NULL, 0, 16, 8, speed_template_16);
2155 		break;
2156 
2157 	case 223:
2158 		test_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec,
2159 				NULL, 0, 16, 16, aead_speed_template_19);
2160 		test_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec,
2161 				NULL, 0, 16, 16, aead_speed_template_19);
2162 		break;
2163 
2164 	case 224:
2165 		test_mb_aead_speed("gcm(sm4)", ENCRYPT, sec, NULL, 0, 16, 8,
2166 				   speed_template_16, num_mb);
2167 		test_mb_aead_speed("gcm(sm4)", DECRYPT, sec, NULL, 0, 16, 8,
2168 				   speed_template_16, num_mb);
2169 		break;
2170 
2171 	case 225:
2172 		test_mb_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec, NULL, 0,
2173 				   16, 16, aead_speed_template_19, num_mb);
2174 		test_mb_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec, NULL, 0,
2175 				   16, 16, aead_speed_template_19, num_mb);
2176 		break;
2177 
2178 	case 226:
2179 		test_cipher_speed("hctr2(aes)", ENCRYPT, sec, NULL,
2180 				  0, speed_template_32);
2181 		break;
2182 
2183 	case 227:
2184 		test_cipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2185 				  speed_template_16_24_32);
2186 		test_cipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2187 				  speed_template_16_24_32);
2188 		test_cipher_speed("cbc(aria)", ENCRYPT, sec, NULL, 0,
2189 				  speed_template_16_24_32);
2190 		test_cipher_speed("cbc(aria)", DECRYPT, sec, NULL, 0,
2191 				  speed_template_16_24_32);
2192 		test_cipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2193 				  speed_template_16_24_32);
2194 		test_cipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2195 				  speed_template_16_24_32);
2196 		break;
2197 
2198 	case 228:
2199 		test_aead_speed("gcm(aria)", ENCRYPT, sec,
2200 				NULL, 0, 16, 8, speed_template_16_24_32);
2201 		test_aead_speed("gcm(aria)", DECRYPT, sec,
2202 				NULL, 0, 16, 8, speed_template_16_24_32);
2203 		break;
2204 
2205 	case 229:
2206 		test_mb_aead_speed("gcm(aria)", ENCRYPT, sec, NULL, 0, 16, 8,
2207 				   speed_template_16, num_mb);
2208 		test_mb_aead_speed("gcm(aria)", DECRYPT, sec, NULL, 0, 16, 8,
2209 				   speed_template_16, num_mb);
2210 		break;
2211 
2212 	case 300:
2213 		if (alg) {
2214 			test_hash_speed(alg, sec, generic_hash_speed_template);
2215 			break;
2216 		}
2217 		fallthrough;
2218 	case 301:
2219 		test_hash_speed("md4", sec, generic_hash_speed_template);
2220 		if (mode > 300 && mode < 400) break;
2221 		fallthrough;
2222 	case 302:
2223 		test_hash_speed("md5", sec, generic_hash_speed_template);
2224 		if (mode > 300 && mode < 400) break;
2225 		fallthrough;
2226 	case 303:
2227 		test_hash_speed("sha1", sec, generic_hash_speed_template);
2228 		if (mode > 300 && mode < 400) break;
2229 		fallthrough;
2230 	case 304:
2231 		test_hash_speed("sha256", sec, generic_hash_speed_template);
2232 		if (mode > 300 && mode < 400) break;
2233 		fallthrough;
2234 	case 305:
2235 		test_hash_speed("sha384", sec, generic_hash_speed_template);
2236 		if (mode > 300 && mode < 400) break;
2237 		fallthrough;
2238 	case 306:
2239 		test_hash_speed("sha512", sec, generic_hash_speed_template);
2240 		if (mode > 300 && mode < 400) break;
2241 		fallthrough;
2242 	case 307:
2243 		test_hash_speed("wp256", sec, generic_hash_speed_template);
2244 		if (mode > 300 && mode < 400) break;
2245 		fallthrough;
2246 	case 308:
2247 		test_hash_speed("wp384", sec, generic_hash_speed_template);
2248 		if (mode > 300 && mode < 400) break;
2249 		fallthrough;
2250 	case 309:
2251 		test_hash_speed("wp512", sec, generic_hash_speed_template);
2252 		if (mode > 300 && mode < 400) break;
2253 		fallthrough;
2254 	case 313:
2255 		test_hash_speed("sha224", sec, generic_hash_speed_template);
2256 		if (mode > 300 && mode < 400) break;
2257 		fallthrough;
2258 	case 314:
2259 		test_hash_speed("xxhash64", sec, generic_hash_speed_template);
2260 		if (mode > 300 && mode < 400) break;
2261 		fallthrough;
2262 	case 315:
2263 		test_hash_speed("rmd160", sec, generic_hash_speed_template);
2264 		if (mode > 300 && mode < 400) break;
2265 		fallthrough;
2266 	case 317:
2267 		test_hash_speed("blake2b-512", sec, generic_hash_speed_template);
2268 		if (mode > 300 && mode < 400) break;
2269 		fallthrough;
2270 	case 318:
2271 		klen = 16;
2272 		test_hash_speed("ghash", sec, generic_hash_speed_template);
2273 		if (mode > 300 && mode < 400) break;
2274 		fallthrough;
2275 	case 319:
2276 		test_hash_speed("crc32c", sec, generic_hash_speed_template);
2277 		if (mode > 300 && mode < 400) break;
2278 		fallthrough;
2279 	case 320:
2280 		test_hash_speed("crct10dif", sec, generic_hash_speed_template);
2281 		if (mode > 300 && mode < 400) break;
2282 		fallthrough;
2283 	case 321:
2284 		test_hash_speed("poly1305", sec, poly1305_speed_template);
2285 		if (mode > 300 && mode < 400) break;
2286 		fallthrough;
2287 	case 322:
2288 		test_hash_speed("sha3-224", sec, generic_hash_speed_template);
2289 		if (mode > 300 && mode < 400) break;
2290 		fallthrough;
2291 	case 323:
2292 		test_hash_speed("sha3-256", sec, generic_hash_speed_template);
2293 		if (mode > 300 && mode < 400) break;
2294 		fallthrough;
2295 	case 324:
2296 		test_hash_speed("sha3-384", sec, generic_hash_speed_template);
2297 		if (mode > 300 && mode < 400) break;
2298 		fallthrough;
2299 	case 325:
2300 		test_hash_speed("sha3-512", sec, generic_hash_speed_template);
2301 		if (mode > 300 && mode < 400) break;
2302 		fallthrough;
2303 	case 326:
2304 		test_hash_speed("sm3", sec, generic_hash_speed_template);
2305 		if (mode > 300 && mode < 400) break;
2306 		fallthrough;
2307 	case 327:
2308 		test_hash_speed("streebog256", sec,
2309 				generic_hash_speed_template);
2310 		if (mode > 300 && mode < 400) break;
2311 		fallthrough;
2312 	case 328:
2313 		test_hash_speed("streebog512", sec,
2314 				generic_hash_speed_template);
2315 		if (mode > 300 && mode < 400) break;
2316 		fallthrough;
2317 	case 399:
2318 		break;
2319 
2320 	case 400:
2321 		if (alg) {
2322 			test_ahash_speed(alg, sec, generic_hash_speed_template);
2323 			break;
2324 		}
2325 		fallthrough;
2326 	case 401:
2327 		test_ahash_speed("md4", sec, generic_hash_speed_template);
2328 		if (mode > 400 && mode < 500) break;
2329 		fallthrough;
2330 	case 402:
2331 		test_ahash_speed("md5", sec, generic_hash_speed_template);
2332 		if (mode > 400 && mode < 500) break;
2333 		fallthrough;
2334 	case 403:
2335 		test_ahash_speed("sha1", sec, generic_hash_speed_template);
2336 		if (mode > 400 && mode < 500) break;
2337 		fallthrough;
2338 	case 404:
2339 		test_ahash_speed("sha256", sec, generic_hash_speed_template);
2340 		if (mode > 400 && mode < 500) break;
2341 		fallthrough;
2342 	case 405:
2343 		test_ahash_speed("sha384", sec, generic_hash_speed_template);
2344 		if (mode > 400 && mode < 500) break;
2345 		fallthrough;
2346 	case 406:
2347 		test_ahash_speed("sha512", sec, generic_hash_speed_template);
2348 		if (mode > 400 && mode < 500) break;
2349 		fallthrough;
2350 	case 407:
2351 		test_ahash_speed("wp256", sec, generic_hash_speed_template);
2352 		if (mode > 400 && mode < 500) break;
2353 		fallthrough;
2354 	case 408:
2355 		test_ahash_speed("wp384", sec, generic_hash_speed_template);
2356 		if (mode > 400 && mode < 500) break;
2357 		fallthrough;
2358 	case 409:
2359 		test_ahash_speed("wp512", sec, generic_hash_speed_template);
2360 		if (mode > 400 && mode < 500) break;
2361 		fallthrough;
2362 	case 413:
2363 		test_ahash_speed("sha224", sec, generic_hash_speed_template);
2364 		if (mode > 400 && mode < 500) break;
2365 		fallthrough;
2366 	case 414:
2367 		test_ahash_speed("xxhash64", sec, generic_hash_speed_template);
2368 		if (mode > 400 && mode < 500) break;
2369 		fallthrough;
2370 	case 415:
2371 		test_ahash_speed("rmd160", sec, generic_hash_speed_template);
2372 		if (mode > 400 && mode < 500) break;
2373 		fallthrough;
2374 	case 417:
2375 		test_ahash_speed("blake2b-512", sec, generic_hash_speed_template);
2376 		if (mode > 400 && mode < 500) break;
2377 		fallthrough;
2378 	case 418:
2379 		test_ahash_speed("sha3-224", sec, generic_hash_speed_template);
2380 		if (mode > 400 && mode < 500) break;
2381 		fallthrough;
2382 	case 419:
2383 		test_ahash_speed("sha3-256", sec, generic_hash_speed_template);
2384 		if (mode > 400 && mode < 500) break;
2385 		fallthrough;
2386 	case 420:
2387 		test_ahash_speed("sha3-384", sec, generic_hash_speed_template);
2388 		if (mode > 400 && mode < 500) break;
2389 		fallthrough;
2390 	case 421:
2391 		test_ahash_speed("sha3-512", sec, generic_hash_speed_template);
2392 		if (mode > 400 && mode < 500) break;
2393 		fallthrough;
2394 	case 422:
2395 		test_ahash_speed("sm3", sec, generic_hash_speed_template);
2396 		if (mode > 400 && mode < 500) break;
2397 		fallthrough;
2398 	case 499:
2399 		break;
2400 
2401 	case 500:
2402 		test_acipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2403 				   speed_template_16_24_32);
2404 		test_acipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2405 				   speed_template_16_24_32);
2406 		test_acipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2407 				   speed_template_16_24_32);
2408 		test_acipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2409 				   speed_template_16_24_32);
2410 		test_acipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2411 				   speed_template_32_40_48);
2412 		test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2413 				   speed_template_32_40_48);
2414 		test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2415 				   speed_template_32_64);
2416 		test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2417 				   speed_template_32_64);
2418 		test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2419 				   speed_template_16_24_32);
2420 		test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2421 				   speed_template_16_24_32);
2422 		test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2423 				   speed_template_16_24_32);
2424 		test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2425 				   speed_template_16_24_32);
2426 		test_acipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, 0,
2427 				   speed_template_20_28_36);
2428 		test_acipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, 0,
2429 				   speed_template_20_28_36);
2430 		break;
2431 
2432 	case 501:
2433 		test_acipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2434 				   des3_speed_template, DES3_SPEED_VECTORS,
2435 				   speed_template_24);
2436 		test_acipher_speed("ecb(des3_ede)", DECRYPT, sec,
2437 				   des3_speed_template, DES3_SPEED_VECTORS,
2438 				   speed_template_24);
2439 		test_acipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2440 				   des3_speed_template, DES3_SPEED_VECTORS,
2441 				   speed_template_24);
2442 		test_acipher_speed("cbc(des3_ede)", DECRYPT, sec,
2443 				   des3_speed_template, DES3_SPEED_VECTORS,
2444 				   speed_template_24);
2445 		break;
2446 
2447 	case 502:
2448 		test_acipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2449 				   speed_template_8);
2450 		test_acipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2451 				   speed_template_8);
2452 		test_acipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2453 				   speed_template_8);
2454 		test_acipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2455 				   speed_template_8);
2456 		break;
2457 
2458 	case 503:
2459 		test_acipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2460 				   speed_template_16_32);
2461 		test_acipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2462 				   speed_template_16_32);
2463 		test_acipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2464 				   speed_template_16_32);
2465 		test_acipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2466 				   speed_template_16_32);
2467 		test_acipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2468 				   speed_template_16_32);
2469 		test_acipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2470 				   speed_template_16_32);
2471 		test_acipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2472 				   speed_template_32_48);
2473 		test_acipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2474 				   speed_template_32_48);
2475 		test_acipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2476 				   speed_template_32_64);
2477 		test_acipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2478 				   speed_template_32_64);
2479 		break;
2480 
2481 	case 504:
2482 		test_acipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2483 				   speed_template_16_24_32);
2484 		test_acipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2485 				   speed_template_16_24_32);
2486 		test_acipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2487 				   speed_template_16_24_32);
2488 		test_acipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2489 				   speed_template_16_24_32);
2490 		test_acipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2491 				   speed_template_16_24_32);
2492 		test_acipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2493 				   speed_template_16_24_32);
2494 		test_acipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2495 				   speed_template_32_40_48);
2496 		test_acipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2497 				   speed_template_32_40_48);
2498 		test_acipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2499 				   speed_template_32_48_64);
2500 		test_acipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2501 				   speed_template_32_48_64);
2502 		break;
2503 
2504 	case 505:
2505 		test_acipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2506 				   speed_template_8);
2507 		break;
2508 
2509 	case 506:
2510 		test_acipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2511 				   speed_template_8_16);
2512 		test_acipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2513 				   speed_template_8_16);
2514 		test_acipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2515 				   speed_template_8_16);
2516 		test_acipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2517 				   speed_template_8_16);
2518 		test_acipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2519 				   speed_template_8_16);
2520 		test_acipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2521 				   speed_template_8_16);
2522 		break;
2523 
2524 	case 507:
2525 		test_acipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2526 				   speed_template_16_32);
2527 		test_acipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2528 				   speed_template_16_32);
2529 		test_acipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2530 				   speed_template_16_32);
2531 		test_acipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2532 				   speed_template_16_32);
2533 		test_acipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2534 				   speed_template_16_32);
2535 		test_acipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2536 				   speed_template_16_32);
2537 		test_acipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2538 				   speed_template_32_48);
2539 		test_acipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2540 				   speed_template_32_48);
2541 		test_acipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2542 				   speed_template_32_64);
2543 		test_acipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2544 				   speed_template_32_64);
2545 		break;
2546 
2547 	case 508:
2548 		test_acipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2549 				   speed_template_16_32);
2550 		test_acipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2551 				   speed_template_16_32);
2552 		test_acipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2553 				   speed_template_16_32);
2554 		test_acipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2555 				   speed_template_16_32);
2556 		test_acipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2557 				   speed_template_16_32);
2558 		test_acipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
2559 				   speed_template_16_32);
2560 		test_acipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
2561 				   speed_template_32_48);
2562 		test_acipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
2563 				   speed_template_32_48);
2564 		test_acipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
2565 				   speed_template_32_64);
2566 		test_acipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2567 				   speed_template_32_64);
2568 		break;
2569 
2570 	case 509:
2571 		test_acipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
2572 				   speed_template_8_32);
2573 		test_acipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
2574 				   speed_template_8_32);
2575 		test_acipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
2576 				   speed_template_8_32);
2577 		test_acipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
2578 				   speed_template_8_32);
2579 		test_acipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
2580 				   speed_template_8_32);
2581 		test_acipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
2582 				   speed_template_8_32);
2583 		break;
2584 
2585 	case 518:
2586 		test_acipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
2587 				speed_template_16);
2588 		test_acipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
2589 				speed_template_16);
2590 		test_acipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
2591 				speed_template_16);
2592 		test_acipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
2593 				speed_template_16);
2594 		test_acipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
2595 				speed_template_16);
2596 		test_acipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
2597 				speed_template_16);
2598 		test_acipher_speed("xts(sm4)", ENCRYPT, sec, NULL, 0,
2599 				speed_template_32);
2600 		test_acipher_speed("xts(sm4)", DECRYPT, sec, NULL, 0,
2601 				speed_template_32);
2602 		break;
2603 
2604 	case 519:
2605 		test_acipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2606 				   speed_template_16_24_32);
2607 		test_acipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2608 				   speed_template_16_24_32);
2609 		test_acipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2610 				   speed_template_16_24_32);
2611 		test_acipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2612 				   speed_template_16_24_32);
2613 		break;
2614 
2615 	case 600:
2616 		if (alg) {
2617 			u8 speed_template[2] = {klen, 0};
2618 			test_mb_skcipher_speed(alg, ENCRYPT, sec, NULL, 0,
2619 					       speed_template, num_mb);
2620 			test_mb_skcipher_speed(alg, DECRYPT, sec, NULL, 0,
2621 					       speed_template, num_mb);
2622 			break;
2623 		}
2624 
2625 		test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2626 				       speed_template_16_24_32, num_mb);
2627 		test_mb_skcipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2628 				       speed_template_16_24_32, num_mb);
2629 		test_mb_skcipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2630 				       speed_template_16_24_32, num_mb);
2631 		test_mb_skcipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2632 				       speed_template_16_24_32, num_mb);
2633 		test_mb_skcipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2634 				       speed_template_32_40_48, num_mb);
2635 		test_mb_skcipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2636 				       speed_template_32_40_48, num_mb);
2637 		test_mb_skcipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2638 				       speed_template_32_64, num_mb);
2639 		test_mb_skcipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2640 				       speed_template_32_64, num_mb);
2641 		test_mb_skcipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2642 				       speed_template_16_24_32, num_mb);
2643 		test_mb_skcipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2644 				       speed_template_16_24_32, num_mb);
2645 		test_mb_skcipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2646 				       speed_template_16_24_32, num_mb);
2647 		test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2648 				       speed_template_16_24_32, num_mb);
2649 		test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL,
2650 				       0, speed_template_20_28_36, num_mb);
2651 		test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL,
2652 				       0, speed_template_20_28_36, num_mb);
2653 		break;
2654 
2655 	case 601:
2656 		test_mb_skcipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2657 				       des3_speed_template, DES3_SPEED_VECTORS,
2658 				       speed_template_24, num_mb);
2659 		test_mb_skcipher_speed("ecb(des3_ede)", DECRYPT, sec,
2660 				       des3_speed_template, DES3_SPEED_VECTORS,
2661 				       speed_template_24, num_mb);
2662 		test_mb_skcipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2663 				       des3_speed_template, DES3_SPEED_VECTORS,
2664 				       speed_template_24, num_mb);
2665 		test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec,
2666 				       des3_speed_template, DES3_SPEED_VECTORS,
2667 				       speed_template_24, num_mb);
2668 		break;
2669 
2670 	case 602:
2671 		test_mb_skcipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2672 				       speed_template_8, num_mb);
2673 		test_mb_skcipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2674 				       speed_template_8, num_mb);
2675 		test_mb_skcipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2676 				       speed_template_8, num_mb);
2677 		test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2678 				       speed_template_8, num_mb);
2679 		break;
2680 
2681 	case 603:
2682 		test_mb_skcipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2683 				       speed_template_16_32, num_mb);
2684 		test_mb_skcipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2685 				       speed_template_16_32, num_mb);
2686 		test_mb_skcipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2687 				       speed_template_16_32, num_mb);
2688 		test_mb_skcipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2689 				       speed_template_16_32, num_mb);
2690 		test_mb_skcipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2691 				       speed_template_16_32, num_mb);
2692 		test_mb_skcipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2693 				       speed_template_16_32, num_mb);
2694 		test_mb_skcipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2695 				       speed_template_32_48, num_mb);
2696 		test_mb_skcipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2697 				       speed_template_32_48, num_mb);
2698 		test_mb_skcipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2699 				       speed_template_32_64, num_mb);
2700 		test_mb_skcipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2701 				       speed_template_32_64, num_mb);
2702 		break;
2703 
2704 	case 604:
2705 		test_mb_skcipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2706 				       speed_template_16_24_32, num_mb);
2707 		test_mb_skcipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2708 				       speed_template_16_24_32, num_mb);
2709 		test_mb_skcipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2710 				       speed_template_16_24_32, num_mb);
2711 		test_mb_skcipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2712 				       speed_template_16_24_32, num_mb);
2713 		test_mb_skcipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2714 				       speed_template_16_24_32, num_mb);
2715 		test_mb_skcipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2716 				       speed_template_16_24_32, num_mb);
2717 		test_mb_skcipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2718 				       speed_template_32_40_48, num_mb);
2719 		test_mb_skcipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2720 				       speed_template_32_40_48, num_mb);
2721 		test_mb_skcipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2722 				       speed_template_32_48_64, num_mb);
2723 		test_mb_skcipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2724 				       speed_template_32_48_64, num_mb);
2725 		break;
2726 
2727 	case 605:
2728 		test_mb_skcipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2729 				       speed_template_8, num_mb);
2730 		break;
2731 
2732 	case 606:
2733 		test_mb_skcipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2734 				       speed_template_8_16, num_mb);
2735 		test_mb_skcipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2736 				       speed_template_8_16, num_mb);
2737 		test_mb_skcipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2738 				       speed_template_8_16, num_mb);
2739 		test_mb_skcipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2740 				       speed_template_8_16, num_mb);
2741 		test_mb_skcipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2742 				       speed_template_8_16, num_mb);
2743 		test_mb_skcipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2744 				       speed_template_8_16, num_mb);
2745 		break;
2746 
2747 	case 607:
2748 		test_mb_skcipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2749 				       speed_template_16_32, num_mb);
2750 		test_mb_skcipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2751 				       speed_template_16_32, num_mb);
2752 		test_mb_skcipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2753 				       speed_template_16_32, num_mb);
2754 		test_mb_skcipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2755 				       speed_template_16_32, num_mb);
2756 		test_mb_skcipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2757 				       speed_template_16_32, num_mb);
2758 		test_mb_skcipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2759 				       speed_template_16_32, num_mb);
2760 		test_mb_skcipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2761 				       speed_template_32_48, num_mb);
2762 		test_mb_skcipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2763 				       speed_template_32_48, num_mb);
2764 		test_mb_skcipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2765 				       speed_template_32_64, num_mb);
2766 		test_mb_skcipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2767 				       speed_template_32_64, num_mb);
2768 		break;
2769 
2770 	case 608:
2771 		test_mb_skcipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2772 				       speed_template_16_32, num_mb);
2773 		test_mb_skcipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2774 				       speed_template_16_32, num_mb);
2775 		test_mb_skcipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2776 				       speed_template_16_32, num_mb);
2777 		test_mb_skcipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2778 				       speed_template_16_32, num_mb);
2779 		test_mb_skcipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2780 				       speed_template_16_32, num_mb);
2781 		test_mb_skcipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
2782 				       speed_template_16_32, num_mb);
2783 		test_mb_skcipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
2784 				       speed_template_32_48, num_mb);
2785 		test_mb_skcipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
2786 				       speed_template_32_48, num_mb);
2787 		test_mb_skcipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
2788 				       speed_template_32_64, num_mb);
2789 		test_mb_skcipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2790 				       speed_template_32_64, num_mb);
2791 		break;
2792 
2793 	case 609:
2794 		test_mb_skcipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
2795 				       speed_template_8_32, num_mb);
2796 		test_mb_skcipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
2797 				       speed_template_8_32, num_mb);
2798 		test_mb_skcipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
2799 				       speed_template_8_32, num_mb);
2800 		test_mb_skcipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
2801 				       speed_template_8_32, num_mb);
2802 		test_mb_skcipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
2803 				       speed_template_8_32, num_mb);
2804 		test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
2805 				       speed_template_8_32, num_mb);
2806 		break;
2807 
2808 	case 610:
2809 		test_mb_skcipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2810 				       speed_template_16_32, num_mb);
2811 		test_mb_skcipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2812 				       speed_template_16_32, num_mb);
2813 		test_mb_skcipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2814 				       speed_template_16_32, num_mb);
2815 		test_mb_skcipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2816 				       speed_template_16_32, num_mb);
2817 		break;
2818 
2819 	}
2820 
2821 	return ret;
2822 }
2823 
tcrypt_mod_init(void)2824 static int __init tcrypt_mod_init(void)
2825 {
2826 	int err = -ENOMEM;
2827 	int i;
2828 
2829 	for (i = 0; i < TVMEMSIZE; i++) {
2830 		tvmem[i] = (void *)__get_free_page(GFP_KERNEL);
2831 		if (!tvmem[i])
2832 			goto err_free_tv;
2833 	}
2834 
2835 	err = do_test(alg, type, mask, mode, num_mb);
2836 
2837 	if (err) {
2838 		pr_err("one or more tests failed!\n");
2839 		goto err_free_tv;
2840 	} else {
2841 		pr_debug("all tests passed\n");
2842 	}
2843 
2844 	/* We intentionaly return -EAGAIN to prevent keeping the module,
2845 	 * unless we're running in fips mode. It does all its work from
2846 	 * init() and doesn't offer any runtime functionality, but in
2847 	 * the fips case, checking for a successful load is helpful.
2848 	 * => we don't need it in the memory, do we?
2849 	 *                                        -- mludvig
2850 	 */
2851 	if (!fips_enabled)
2852 		err = -EAGAIN;
2853 
2854 err_free_tv:
2855 	for (i = 0; i < TVMEMSIZE && tvmem[i]; i++)
2856 		free_page((unsigned long)tvmem[i]);
2857 
2858 	return err;
2859 }
2860 
2861 /*
2862  * If an init function is provided, an exit function must also be provided
2863  * to allow module unload.
2864  */
tcrypt_mod_fini(void)2865 static void __exit tcrypt_mod_fini(void) { }
2866 
2867 late_initcall(tcrypt_mod_init);
2868 module_exit(tcrypt_mod_fini);
2869 
2870 module_param(alg, charp, 0);
2871 module_param(type, uint, 0);
2872 module_param(mask, uint, 0);
2873 module_param(mode, int, 0);
2874 module_param(sec, uint, 0);
2875 MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
2876 		      "(defaults to zero which uses CPU cycles instead)");
2877 module_param(num_mb, uint, 0000);
2878 MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)");
2879 module_param(klen, uint, 0);
2880 MODULE_PARM_DESC(klen, "Key length (defaults to 0)");
2881 
2882 MODULE_LICENSE("GPL");
2883 MODULE_DESCRIPTION("Quick & dirty crypto testing module");
2884 MODULE_AUTHOR("James Morris <jmorris@intercode.com.au>");
2885