xref: /linux/crypto/algapi.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Cryptographic API for algorithms (i.e., low-level API).
4  *
5  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
6  */
7 
8 #include <crypto/algapi.h>
9 #include <linux/err.h>
10 #include <linux/errno.h>
11 #include <linux/fips.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/list.h>
15 #include <linux/module.h>
16 #include <linux/rtnetlink.h>
17 #include <linux/slab.h>
18 #include <linux/string.h>
19 #include <linux/workqueue.h>
20 
21 #include "internal.h"
22 
23 static LIST_HEAD(crypto_template_list);
24 
25 static inline void crypto_check_module_sig(struct module *mod)
26 {
27 	if (fips_enabled && mod && !module_sig_ok(mod))
28 		panic("Module %s signature verification failed in FIPS mode\n",
29 		      module_name(mod));
30 }
31 
32 static int crypto_check_alg(struct crypto_alg *alg)
33 {
34 	crypto_check_module_sig(alg->cra_module);
35 
36 	if (!alg->cra_name[0] || !alg->cra_driver_name[0])
37 		return -EINVAL;
38 
39 	if (alg->cra_alignmask & (alg->cra_alignmask + 1))
40 		return -EINVAL;
41 
42 	/* General maximums for all algs. */
43 	if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)
44 		return -EINVAL;
45 
46 	if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)
47 		return -EINVAL;
48 
49 	/* Lower maximums for specific alg types. */
50 	if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
51 			       CRYPTO_ALG_TYPE_CIPHER) {
52 		if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
53 			return -EINVAL;
54 
55 		if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)
56 			return -EINVAL;
57 	}
58 
59 	if (alg->cra_priority < 0)
60 		return -EINVAL;
61 
62 	refcount_set(&alg->cra_refcnt, 1);
63 
64 	return 0;
65 }
66 
67 static void crypto_free_instance(struct crypto_instance *inst)
68 {
69 	inst->alg.cra_type->free(inst);
70 }
71 
72 static void crypto_destroy_instance_workfn(struct work_struct *w)
73 {
74 	struct crypto_template *tmpl = container_of(w, struct crypto_template,
75 						    free_work);
76 	struct crypto_instance *inst;
77 	struct hlist_node *n;
78 	HLIST_HEAD(list);
79 
80 	down_write(&crypto_alg_sem);
81 	hlist_for_each_entry_safe(inst, n, &tmpl->dead, list) {
82 		if (refcount_read(&inst->alg.cra_refcnt) != -1)
83 			continue;
84 		hlist_del(&inst->list);
85 		hlist_add_head(&inst->list, &list);
86 	}
87 	up_write(&crypto_alg_sem);
88 
89 	hlist_for_each_entry_safe(inst, n, &list, list)
90 		crypto_free_instance(inst);
91 }
92 
93 static void crypto_destroy_instance(struct crypto_alg *alg)
94 {
95 	struct crypto_instance *inst = container_of(alg,
96 						    struct crypto_instance,
97 						    alg);
98 	struct crypto_template *tmpl = inst->tmpl;
99 
100 	refcount_set(&alg->cra_refcnt, -1);
101 	schedule_work(&tmpl->free_work);
102 }
103 
104 /*
105  * This function adds a spawn to the list secondary_spawns which
106  * will be used at the end of crypto_remove_spawns to unregister
107  * instances, unless the spawn happens to be one that is depended
108  * on by the new algorithm (nalg in crypto_remove_spawns).
109  *
110  * This function is also responsible for resurrecting any algorithms
111  * in the dependency chain of nalg by unsetting n->dead.
112  */
113 static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
114 					    struct list_head *stack,
115 					    struct list_head *top,
116 					    struct list_head *secondary_spawns)
117 {
118 	struct crypto_spawn *spawn, *n;
119 
120 	spawn = list_first_entry_or_null(stack, struct crypto_spawn, list);
121 	if (!spawn)
122 		return NULL;
123 
124 	n = list_prev_entry(spawn, list);
125 	list_move(&spawn->list, secondary_spawns);
126 
127 	if (list_is_last(&n->list, stack))
128 		return top;
129 
130 	n = list_next_entry(n, list);
131 	if (!spawn->dead)
132 		n->dead = false;
133 
134 	return &n->inst->alg.cra_users;
135 }
136 
137 static void crypto_remove_instance(struct crypto_instance *inst,
138 				   struct list_head *list)
139 {
140 	struct crypto_template *tmpl = inst->tmpl;
141 
142 	if (crypto_is_dead(&inst->alg))
143 		return;
144 
145 	inst->alg.cra_flags |= CRYPTO_ALG_DEAD;
146 
147 	if (!tmpl)
148 		return;
149 
150 	list_del_init(&inst->alg.cra_list);
151 	hlist_del(&inst->list);
152 	hlist_add_head(&inst->list, &tmpl->dead);
153 
154 	BUG_ON(!list_empty(&inst->alg.cra_users));
155 
156 	crypto_alg_put(&inst->alg);
157 }
158 
159 /*
160  * Given an algorithm alg, remove all algorithms that depend on it
161  * through spawns.  If nalg is not null, then exempt any algorithms
162  * that is depended on by nalg.  This is useful when nalg itself
163  * depends on alg.
164  */
165 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
166 			  struct crypto_alg *nalg)
167 {
168 	u32 new_type = (nalg ?: alg)->cra_flags;
169 	struct crypto_spawn *spawn, *n;
170 	LIST_HEAD(secondary_spawns);
171 	struct list_head *spawns;
172 	LIST_HEAD(stack);
173 	LIST_HEAD(top);
174 
175 	spawns = &alg->cra_users;
176 	list_for_each_entry_safe(spawn, n, spawns, list) {
177 		if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
178 			continue;
179 
180 		list_move(&spawn->list, &top);
181 	}
182 
183 	/*
184 	 * Perform a depth-first walk starting from alg through
185 	 * the cra_users tree.  The list stack records the path
186 	 * from alg to the current spawn.
187 	 */
188 	spawns = &top;
189 	do {
190 		while (!list_empty(spawns)) {
191 			struct crypto_instance *inst;
192 
193 			spawn = list_first_entry(spawns, struct crypto_spawn,
194 						 list);
195 			inst = spawn->inst;
196 
197 			list_move(&spawn->list, &stack);
198 			spawn->dead = !spawn->registered || &inst->alg != nalg;
199 
200 			if (!spawn->registered)
201 				break;
202 
203 			BUG_ON(&inst->alg == alg);
204 
205 			if (&inst->alg == nalg)
206 				break;
207 
208 			spawns = &inst->alg.cra_users;
209 
210 			/*
211 			 * Even if spawn->registered is true, the
212 			 * instance itself may still be unregistered.
213 			 * This is because it may have failed during
214 			 * registration.  Therefore we still need to
215 			 * make the following test.
216 			 *
217 			 * We may encounter an unregistered instance here, since
218 			 * an instance's spawns are set up prior to the instance
219 			 * being registered.  An unregistered instance will have
220 			 * NULL ->cra_users.next, since ->cra_users isn't
221 			 * properly initialized until registration.  But an
222 			 * unregistered instance cannot have any users, so treat
223 			 * it the same as ->cra_users being empty.
224 			 */
225 			if (spawns->next == NULL)
226 				break;
227 		}
228 	} while ((spawns = crypto_more_spawns(alg, &stack, &top,
229 					      &secondary_spawns)));
230 
231 	/*
232 	 * Remove all instances that are marked as dead.  Also
233 	 * complete the resurrection of the others by moving them
234 	 * back to the cra_users list.
235 	 */
236 	list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
237 		if (!spawn->dead)
238 			list_move(&spawn->list, &spawn->alg->cra_users);
239 		else if (spawn->registered)
240 			crypto_remove_instance(spawn->inst, list);
241 	}
242 }
243 EXPORT_SYMBOL_GPL(crypto_remove_spawns);
244 
245 static void crypto_alg_finish_registration(struct crypto_alg *alg,
246 					   struct list_head *algs_to_put)
247 	__must_hold(&crypto_alg_sem)
248 {
249 	struct crypto_alg *q;
250 
251 	list_for_each_entry(q, &crypto_alg_list, cra_list) {
252 		if (q == alg)
253 			continue;
254 
255 		if (crypto_is_moribund(q))
256 			continue;
257 
258 		if (crypto_is_larval(q))
259 			continue;
260 
261 		if (strcmp(alg->cra_name, q->cra_name))
262 			continue;
263 
264 		if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
265 		    q->cra_priority > alg->cra_priority)
266 			continue;
267 
268 		crypto_remove_spawns(q, algs_to_put, alg);
269 	}
270 
271 	crypto_notify(CRYPTO_MSG_ALG_LOADED, alg);
272 }
273 
274 static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)
275 {
276 	struct crypto_larval *larval;
277 
278 	if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) ||
279 	    (alg->cra_flags & CRYPTO_ALG_INTERNAL))
280 		return NULL; /* No self-test needed */
281 
282 	larval = crypto_larval_alloc(alg->cra_name,
283 				     alg->cra_flags | CRYPTO_ALG_TESTED, 0);
284 	if (IS_ERR(larval))
285 		return larval;
286 
287 	larval->adult = crypto_mod_get(alg);
288 	if (!larval->adult) {
289 		kfree(larval);
290 		return ERR_PTR(-ENOENT);
291 	}
292 
293 	refcount_set(&larval->alg.cra_refcnt, 1);
294 	memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,
295 	       CRYPTO_MAX_ALG_NAME);
296 	larval->alg.cra_priority = alg->cra_priority;
297 
298 	return larval;
299 }
300 
301 static struct crypto_larval *
302 __crypto_register_alg(struct crypto_alg *alg, struct list_head *algs_to_put)
303 	__must_hold(&crypto_alg_sem)
304 {
305 	struct crypto_alg *q;
306 	struct crypto_larval *larval;
307 	int ret = -EAGAIN;
308 
309 	if (crypto_is_dead(alg))
310 		goto err;
311 
312 	INIT_LIST_HEAD(&alg->cra_users);
313 
314 	ret = -EEXIST;
315 
316 	list_for_each_entry(q, &crypto_alg_list, cra_list) {
317 		if (q == alg)
318 			goto err;
319 
320 		if (crypto_is_moribund(q))
321 			continue;
322 
323 		if (crypto_is_larval(q)) {
324 			if (!strcmp(alg->cra_driver_name, q->cra_driver_name))
325 				goto err;
326 			continue;
327 		}
328 
329 		if (!strcmp(q->cra_driver_name, alg->cra_name) ||
330 		    !strcmp(q->cra_driver_name, alg->cra_driver_name) ||
331 		    !strcmp(q->cra_name, alg->cra_driver_name))
332 			goto err;
333 	}
334 
335 	larval = crypto_alloc_test_larval(alg);
336 	if (IS_ERR(larval))
337 		goto out;
338 
339 	list_add(&alg->cra_list, &crypto_alg_list);
340 
341 	if (larval) {
342 		/* No cheating! */
343 		alg->cra_flags &= ~CRYPTO_ALG_TESTED;
344 
345 		list_add(&larval->alg.cra_list, &crypto_alg_list);
346 	} else {
347 		alg->cra_flags |= CRYPTO_ALG_TESTED;
348 		crypto_alg_finish_registration(alg, algs_to_put);
349 	}
350 
351 out:
352 	return larval;
353 
354 err:
355 	larval = ERR_PTR(ret);
356 	goto out;
357 }
358 
359 void crypto_alg_tested(const char *name, int err)
360 {
361 	struct crypto_larval *test;
362 	struct crypto_alg *alg;
363 	struct crypto_alg *q;
364 	LIST_HEAD(list);
365 
366 	down_write(&crypto_alg_sem);
367 	list_for_each_entry(q, &crypto_alg_list, cra_list) {
368 		if (crypto_is_moribund(q) || !crypto_is_larval(q))
369 			continue;
370 
371 		test = (struct crypto_larval *)q;
372 
373 		if (!strcmp(q->cra_driver_name, name))
374 			goto found;
375 	}
376 
377 	pr_err("alg: Unexpected test result for %s: %d\n", name, err);
378 	up_write(&crypto_alg_sem);
379 	return;
380 
381 found:
382 	q->cra_flags |= CRYPTO_ALG_DEAD;
383 	alg = test->adult;
384 
385 	if (crypto_is_dead(alg))
386 		goto complete;
387 
388 	if (err == -ECANCELED)
389 		alg->cra_flags |= CRYPTO_ALG_FIPS_INTERNAL;
390 	else if (err)
391 		goto complete;
392 	else
393 		alg->cra_flags &= ~CRYPTO_ALG_FIPS_INTERNAL;
394 
395 	alg->cra_flags |= CRYPTO_ALG_TESTED;
396 
397 	crypto_alg_finish_registration(alg, &list);
398 
399 complete:
400 	list_del_init(&test->alg.cra_list);
401 	complete_all(&test->completion);
402 
403 	up_write(&crypto_alg_sem);
404 
405 	crypto_alg_put(&test->alg);
406 	crypto_remove_final(&list);
407 }
408 EXPORT_SYMBOL_GPL(crypto_alg_tested);
409 
410 void crypto_remove_final(struct list_head *list)
411 {
412 	struct crypto_alg *alg;
413 	struct crypto_alg *n;
414 
415 	list_for_each_entry_safe(alg, n, list, cra_list) {
416 		list_del_init(&alg->cra_list);
417 		crypto_alg_put(alg);
418 	}
419 }
420 EXPORT_SYMBOL_GPL(crypto_remove_final);
421 
422 static void crypto_free_alg(struct crypto_alg *alg)
423 {
424 	unsigned int algsize = alg->cra_type->algsize;
425 	u8 *p = (u8 *)alg - algsize;
426 
427 	kfree(p);
428 }
429 
430 int crypto_register_alg(struct crypto_alg *alg)
431 {
432 	struct crypto_larval *larval;
433 	bool test_started = false;
434 	LIST_HEAD(algs_to_put);
435 	int err;
436 
437 	alg->cra_flags &= ~CRYPTO_ALG_DEAD;
438 	err = crypto_check_alg(alg);
439 	if (err)
440 		return err;
441 
442 	if (alg->cra_flags & CRYPTO_ALG_DUP_FIRST &&
443 	    !WARN_ON_ONCE(alg->cra_destroy)) {
444 		unsigned int algsize = alg->cra_type->algsize;
445 		u8 *p = (u8 *)alg - algsize;
446 
447 		p = kmemdup(p, algsize + sizeof(*alg), GFP_KERNEL);
448 		if (!p)
449 			return -ENOMEM;
450 
451 		alg = (void *)(p + algsize);
452 		alg->cra_destroy = crypto_free_alg;
453 	}
454 
455 	down_write(&crypto_alg_sem);
456 	larval = __crypto_register_alg(alg, &algs_to_put);
457 	if (!IS_ERR_OR_NULL(larval)) {
458 		test_started = crypto_boot_test_finished();
459 		larval->test_started = test_started;
460 	}
461 	up_write(&crypto_alg_sem);
462 
463 	if (IS_ERR(larval)) {
464 		crypto_alg_put(alg);
465 		return PTR_ERR(larval);
466 	}
467 
468 	if (test_started)
469 		crypto_schedule_test(larval);
470 	else
471 		crypto_remove_final(&algs_to_put);
472 
473 	return 0;
474 }
475 EXPORT_SYMBOL_GPL(crypto_register_alg);
476 
477 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
478 {
479 	if (unlikely(list_empty(&alg->cra_list)))
480 		return -ENOENT;
481 
482 	alg->cra_flags |= CRYPTO_ALG_DEAD;
483 
484 	list_del_init(&alg->cra_list);
485 	crypto_remove_spawns(alg, list, NULL);
486 
487 	return 0;
488 }
489 
490 void crypto_unregister_alg(struct crypto_alg *alg)
491 {
492 	int ret;
493 	LIST_HEAD(list);
494 
495 	down_write(&crypto_alg_sem);
496 	if (alg->cra_flags & CRYPTO_ALG_DUP_FIRST) {
497 		struct crypto_alg *q, *r = NULL;
498 
499 		list_for_each_entry(q, &crypto_alg_list, cra_list) {
500 			if (crypto_is_larval(q))
501 				continue;
502 			if (strcmp(q->cra_driver_name, alg->cra_driver_name))
503 				continue;
504 			r = q;
505 			break;
506 		}
507 
508 		if (!r) {
509 			up_write(&crypto_alg_sem);
510 			ret = -ENOENT;
511 			goto not_found;
512 		}
513 
514 		alg = r;
515 	}
516 
517 	ret = crypto_remove_alg(alg, &list);
518 	up_write(&crypto_alg_sem);
519 
520 not_found:
521 	if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))
522 		return;
523 
524 	WARN_ON(!alg->cra_destroy && refcount_read(&alg->cra_refcnt) != 1);
525 
526 	list_add(&alg->cra_list, &list);
527 	crypto_remove_final(&list);
528 }
529 EXPORT_SYMBOL_GPL(crypto_unregister_alg);
530 
531 int crypto_register_algs(struct crypto_alg *algs, int count)
532 {
533 	int i, ret;
534 
535 	for (i = 0; i < count; i++) {
536 		ret = crypto_register_alg(&algs[i]);
537 		if (ret) {
538 			crypto_unregister_algs(algs, i);
539 			return ret;
540 		}
541 	}
542 
543 	return 0;
544 }
545 EXPORT_SYMBOL_GPL(crypto_register_algs);
546 
547 void crypto_unregister_algs(struct crypto_alg *algs, int count)
548 {
549 	int i;
550 
551 	for (i = count - 1; i >= 0; --i)
552 		crypto_unregister_alg(&algs[i]);
553 }
554 EXPORT_SYMBOL_GPL(crypto_unregister_algs);
555 
556 int crypto_register_template(struct crypto_template *tmpl)
557 {
558 	struct crypto_template *q;
559 	int err = -EEXIST;
560 
561 	INIT_WORK(&tmpl->free_work, crypto_destroy_instance_workfn);
562 
563 	down_write(&crypto_alg_sem);
564 
565 	crypto_check_module_sig(tmpl->module);
566 
567 	list_for_each_entry(q, &crypto_template_list, list) {
568 		if (q == tmpl)
569 			goto out;
570 	}
571 
572 	list_add(&tmpl->list, &crypto_template_list);
573 	err = 0;
574 out:
575 	up_write(&crypto_alg_sem);
576 	return err;
577 }
578 EXPORT_SYMBOL_GPL(crypto_register_template);
579 
580 int crypto_register_templates(struct crypto_template *tmpls, int count)
581 {
582 	int i, err;
583 
584 	for (i = 0; i < count; i++) {
585 		err = crypto_register_template(&tmpls[i]);
586 		if (err)
587 			goto out;
588 	}
589 	return 0;
590 
591 out:
592 	for (--i; i >= 0; --i)
593 		crypto_unregister_template(&tmpls[i]);
594 	return err;
595 }
596 EXPORT_SYMBOL_GPL(crypto_register_templates);
597 
598 void crypto_unregister_template(struct crypto_template *tmpl)
599 {
600 	struct crypto_instance *inst;
601 	struct hlist_node *n;
602 	struct hlist_head *list;
603 	LIST_HEAD(users);
604 
605 	down_write(&crypto_alg_sem);
606 
607 	BUG_ON(list_empty(&tmpl->list));
608 	list_del_init(&tmpl->list);
609 
610 	list = &tmpl->instances;
611 	hlist_for_each_entry(inst, list, list) {
612 		int err = crypto_remove_alg(&inst->alg, &users);
613 
614 		BUG_ON(err);
615 	}
616 
617 	up_write(&crypto_alg_sem);
618 
619 	hlist_for_each_entry_safe(inst, n, list, list) {
620 		BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);
621 		crypto_free_instance(inst);
622 	}
623 	crypto_remove_final(&users);
624 
625 	flush_work(&tmpl->free_work);
626 }
627 EXPORT_SYMBOL_GPL(crypto_unregister_template);
628 
629 void crypto_unregister_templates(struct crypto_template *tmpls, int count)
630 {
631 	int i;
632 
633 	for (i = count - 1; i >= 0; --i)
634 		crypto_unregister_template(&tmpls[i]);
635 }
636 EXPORT_SYMBOL_GPL(crypto_unregister_templates);
637 
638 static struct crypto_template *__crypto_lookup_template(const char *name)
639 {
640 	struct crypto_template *q, *tmpl = NULL;
641 
642 	down_read(&crypto_alg_sem);
643 	list_for_each_entry(q, &crypto_template_list, list) {
644 		if (strcmp(q->name, name))
645 			continue;
646 		if (unlikely(!crypto_tmpl_get(q)))
647 			continue;
648 
649 		tmpl = q;
650 		break;
651 	}
652 	up_read(&crypto_alg_sem);
653 
654 	return tmpl;
655 }
656 
657 struct crypto_template *crypto_lookup_template(const char *name)
658 {
659 	return try_then_request_module(__crypto_lookup_template(name),
660 				       "crypto-%s", name);
661 }
662 EXPORT_SYMBOL_GPL(crypto_lookup_template);
663 
664 int crypto_register_instance(struct crypto_template *tmpl,
665 			     struct crypto_instance *inst)
666 {
667 	struct crypto_larval *larval;
668 	struct crypto_spawn *spawn;
669 	u32 fips_internal = 0;
670 	LIST_HEAD(algs_to_put);
671 	int err;
672 
673 	err = crypto_check_alg(&inst->alg);
674 	if (err)
675 		return err;
676 
677 	inst->alg.cra_module = tmpl->module;
678 	inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
679 	inst->alg.cra_destroy = crypto_destroy_instance;
680 
681 	down_write(&crypto_alg_sem);
682 
683 	larval = ERR_PTR(-EAGAIN);
684 	for (spawn = inst->spawns; spawn;) {
685 		struct crypto_spawn *next;
686 
687 		if (spawn->dead)
688 			goto unlock;
689 
690 		next = spawn->next;
691 		spawn->inst = inst;
692 		spawn->registered = true;
693 
694 		fips_internal |= spawn->alg->cra_flags;
695 
696 		crypto_mod_put(spawn->alg);
697 
698 		spawn = next;
699 	}
700 
701 	inst->alg.cra_flags |= (fips_internal & CRYPTO_ALG_FIPS_INTERNAL);
702 
703 	larval = __crypto_register_alg(&inst->alg, &algs_to_put);
704 	if (IS_ERR(larval))
705 		goto unlock;
706 	else if (larval)
707 		larval->test_started = true;
708 
709 	hlist_add_head(&inst->list, &tmpl->instances);
710 	inst->tmpl = tmpl;
711 
712 unlock:
713 	up_write(&crypto_alg_sem);
714 
715 	if (IS_ERR(larval))
716 		return PTR_ERR(larval);
717 
718 	if (larval)
719 		crypto_schedule_test(larval);
720 	else
721 		crypto_remove_final(&algs_to_put);
722 
723 	return 0;
724 }
725 EXPORT_SYMBOL_GPL(crypto_register_instance);
726 
727 void crypto_unregister_instance(struct crypto_instance *inst)
728 {
729 	LIST_HEAD(list);
730 
731 	down_write(&crypto_alg_sem);
732 
733 	crypto_remove_spawns(&inst->alg, &list, NULL);
734 	crypto_remove_instance(inst, &list);
735 
736 	up_write(&crypto_alg_sem);
737 
738 	crypto_remove_final(&list);
739 }
740 EXPORT_SYMBOL_GPL(crypto_unregister_instance);
741 
742 int crypto_grab_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst,
743 		      const char *name, u32 type, u32 mask)
744 {
745 	struct crypto_alg *alg;
746 	int err = -EAGAIN;
747 
748 	if (WARN_ON_ONCE(inst == NULL))
749 		return -EINVAL;
750 
751 	/* Allow the result of crypto_attr_alg_name() to be passed directly */
752 	if (IS_ERR(name))
753 		return PTR_ERR(name);
754 
755 	alg = crypto_find_alg(name, spawn->frontend,
756 			      type | CRYPTO_ALG_FIPS_INTERNAL, mask);
757 	if (IS_ERR(alg))
758 		return PTR_ERR(alg);
759 
760 	down_write(&crypto_alg_sem);
761 	if (!crypto_is_moribund(alg)) {
762 		list_add(&spawn->list, &alg->cra_users);
763 		spawn->alg = alg;
764 		spawn->mask = mask;
765 		spawn->next = inst->spawns;
766 		inst->spawns = spawn;
767 		inst->alg.cra_flags |=
768 			(alg->cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
769 		err = 0;
770 	}
771 	up_write(&crypto_alg_sem);
772 	if (err)
773 		crypto_mod_put(alg);
774 	return err;
775 }
776 EXPORT_SYMBOL_GPL(crypto_grab_spawn);
777 
778 void crypto_drop_spawn(struct crypto_spawn *spawn)
779 {
780 	if (!spawn->alg) /* not yet initialized? */
781 		return;
782 
783 	down_write(&crypto_alg_sem);
784 	if (!spawn->dead)
785 		list_del(&spawn->list);
786 	up_write(&crypto_alg_sem);
787 
788 	if (!spawn->registered)
789 		crypto_mod_put(spawn->alg);
790 }
791 EXPORT_SYMBOL_GPL(crypto_drop_spawn);
792 
793 static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn)
794 {
795 	struct crypto_alg *alg = ERR_PTR(-EAGAIN);
796 	struct crypto_alg *target;
797 	bool shoot = false;
798 
799 	down_read(&crypto_alg_sem);
800 	if (!spawn->dead) {
801 		alg = spawn->alg;
802 		if (!crypto_mod_get(alg)) {
803 			target = crypto_alg_get(alg);
804 			shoot = true;
805 			alg = ERR_PTR(-EAGAIN);
806 		}
807 	}
808 	up_read(&crypto_alg_sem);
809 
810 	if (shoot) {
811 		crypto_shoot_alg(target);
812 		crypto_alg_put(target);
813 	}
814 
815 	return alg;
816 }
817 
818 struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type,
819 				    u32 mask)
820 {
821 	struct crypto_alg *alg;
822 	struct crypto_tfm *tfm;
823 
824 	alg = crypto_spawn_alg(spawn);
825 	if (IS_ERR(alg))
826 		return ERR_CAST(alg);
827 
828 	tfm = ERR_PTR(-EINVAL);
829 	if (unlikely((alg->cra_flags ^ type) & mask))
830 		goto out_put_alg;
831 
832 	tfm = __crypto_alloc_tfm(alg, type, mask);
833 	if (IS_ERR(tfm))
834 		goto out_put_alg;
835 
836 	return tfm;
837 
838 out_put_alg:
839 	crypto_mod_put(alg);
840 	return tfm;
841 }
842 EXPORT_SYMBOL_GPL(crypto_spawn_tfm);
843 
844 void *crypto_spawn_tfm2(struct crypto_spawn *spawn)
845 {
846 	struct crypto_alg *alg;
847 	struct crypto_tfm *tfm;
848 
849 	alg = crypto_spawn_alg(spawn);
850 	if (IS_ERR(alg))
851 		return ERR_CAST(alg);
852 
853 	tfm = crypto_create_tfm(alg, spawn->frontend);
854 	if (IS_ERR(tfm))
855 		goto out_put_alg;
856 
857 	return tfm;
858 
859 out_put_alg:
860 	crypto_mod_put(alg);
861 	return tfm;
862 }
863 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);
864 
865 int crypto_register_notifier(struct notifier_block *nb)
866 {
867 	return blocking_notifier_chain_register(&crypto_chain, nb);
868 }
869 EXPORT_SYMBOL_GPL(crypto_register_notifier);
870 
871 int crypto_unregister_notifier(struct notifier_block *nb)
872 {
873 	return blocking_notifier_chain_unregister(&crypto_chain, nb);
874 }
875 EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
876 
877 struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb)
878 {
879 	struct rtattr *rta = tb[0];
880 	struct crypto_attr_type *algt;
881 
882 	if (!rta)
883 		return ERR_PTR(-ENOENT);
884 	if (RTA_PAYLOAD(rta) < sizeof(*algt))
885 		return ERR_PTR(-EINVAL);
886 	if (rta->rta_type != CRYPTOA_TYPE)
887 		return ERR_PTR(-EINVAL);
888 
889 	algt = RTA_DATA(rta);
890 
891 	return algt;
892 }
893 EXPORT_SYMBOL_GPL(crypto_get_attr_type);
894 
895 /**
896  * crypto_check_attr_type() - check algorithm type and compute inherited mask
897  * @tb: the template parameters
898  * @type: the algorithm type the template would be instantiated as
899  * @mask_ret: (output) the mask that should be passed to crypto_grab_*()
900  *	      to restrict the flags of any inner algorithms
901  *
902  * Validate that the algorithm type the user requested is compatible with the
903  * one the template would actually be instantiated as.  E.g., if the user is
904  * doing crypto_alloc_shash("cbc(aes)", ...), this would return an error because
905  * the "cbc" template creates an "skcipher" algorithm, not an "shash" algorithm.
906  *
907  * Also compute the mask to use to restrict the flags of any inner algorithms.
908  *
909  * Return: 0 on success; -errno on failure
910  */
911 int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret)
912 {
913 	struct crypto_attr_type *algt;
914 
915 	algt = crypto_get_attr_type(tb);
916 	if (IS_ERR(algt))
917 		return PTR_ERR(algt);
918 
919 	if ((algt->type ^ type) & algt->mask)
920 		return -EINVAL;
921 
922 	*mask_ret = crypto_algt_inherited_mask(algt);
923 	return 0;
924 }
925 EXPORT_SYMBOL_GPL(crypto_check_attr_type);
926 
927 const char *crypto_attr_alg_name(struct rtattr *rta)
928 {
929 	struct crypto_attr_alg *alga;
930 
931 	if (!rta)
932 		return ERR_PTR(-ENOENT);
933 	if (RTA_PAYLOAD(rta) < sizeof(*alga))
934 		return ERR_PTR(-EINVAL);
935 	if (rta->rta_type != CRYPTOA_ALG)
936 		return ERR_PTR(-EINVAL);
937 
938 	alga = RTA_DATA(rta);
939 	alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;
940 
941 	return alga->name;
942 }
943 EXPORT_SYMBOL_GPL(crypto_attr_alg_name);
944 
945 int __crypto_inst_setname(struct crypto_instance *inst, const char *name,
946 			  const char *driver, struct crypto_alg *alg)
947 {
948 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
949 		     alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
950 		return -ENAMETOOLONG;
951 
952 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
953 		     driver, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
954 		return -ENAMETOOLONG;
955 
956 	return 0;
957 }
958 EXPORT_SYMBOL_GPL(__crypto_inst_setname);
959 
960 void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)
961 {
962 	INIT_LIST_HEAD(&queue->list);
963 	queue->backlog = &queue->list;
964 	queue->qlen = 0;
965 	queue->max_qlen = max_qlen;
966 }
967 EXPORT_SYMBOL_GPL(crypto_init_queue);
968 
969 int crypto_enqueue_request(struct crypto_queue *queue,
970 			   struct crypto_async_request *request)
971 {
972 	int err = -EINPROGRESS;
973 
974 	if (unlikely(queue->qlen >= queue->max_qlen)) {
975 		if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
976 			err = -ENOSPC;
977 			goto out;
978 		}
979 		err = -EBUSY;
980 		if (queue->backlog == &queue->list)
981 			queue->backlog = &request->list;
982 	}
983 
984 	queue->qlen++;
985 	list_add_tail(&request->list, &queue->list);
986 
987 out:
988 	return err;
989 }
990 EXPORT_SYMBOL_GPL(crypto_enqueue_request);
991 
992 void crypto_enqueue_request_head(struct crypto_queue *queue,
993 				 struct crypto_async_request *request)
994 {
995 	if (unlikely(queue->qlen >= queue->max_qlen))
996 		queue->backlog = queue->backlog->prev;
997 
998 	queue->qlen++;
999 	list_add(&request->list, &queue->list);
1000 }
1001 EXPORT_SYMBOL_GPL(crypto_enqueue_request_head);
1002 
1003 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
1004 {
1005 	struct list_head *request;
1006 
1007 	if (unlikely(!queue->qlen))
1008 		return NULL;
1009 
1010 	queue->qlen--;
1011 
1012 	if (queue->backlog != &queue->list)
1013 		queue->backlog = queue->backlog->next;
1014 
1015 	request = queue->list.next;
1016 	list_del_init(request);
1017 
1018 	return list_entry(request, struct crypto_async_request, list);
1019 }
1020 EXPORT_SYMBOL_GPL(crypto_dequeue_request);
1021 
1022 static inline void crypto_inc_byte(u8 *a, unsigned int size)
1023 {
1024 	u8 *b = (a + size);
1025 	u8 c;
1026 
1027 	for (; size; size--) {
1028 		c = *--b + 1;
1029 		*b = c;
1030 		if (c)
1031 			break;
1032 	}
1033 }
1034 
1035 void crypto_inc(u8 *a, unsigned int size)
1036 {
1037 	__be32 *b = (__be32 *)(a + size);
1038 	u32 c;
1039 
1040 	if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
1041 	    IS_ALIGNED((unsigned long)b, __alignof__(*b)))
1042 		for (; size >= 4; size -= 4) {
1043 			c = be32_to_cpu(*--b) + 1;
1044 			*b = cpu_to_be32(c);
1045 			if (likely(c))
1046 				return;
1047 		}
1048 
1049 	crypto_inc_byte(a, size);
1050 }
1051 EXPORT_SYMBOL_GPL(crypto_inc);
1052 
1053 unsigned int crypto_alg_extsize(struct crypto_alg *alg)
1054 {
1055 	return alg->cra_ctxsize +
1056 	       (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
1057 }
1058 EXPORT_SYMBOL_GPL(crypto_alg_extsize);
1059 
1060 int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
1061 			u32 type, u32 mask)
1062 {
1063 	int ret = 0;
1064 	struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);
1065 
1066 	if (!IS_ERR(alg)) {
1067 		crypto_mod_put(alg);
1068 		ret = 1;
1069 	}
1070 
1071 	return ret;
1072 }
1073 EXPORT_SYMBOL_GPL(crypto_type_has_alg);
1074 
1075 static void __init crypto_start_tests(void)
1076 {
1077 	if (!IS_BUILTIN(CONFIG_CRYPTO_ALGAPI))
1078 		return;
1079 
1080 	if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS))
1081 		return;
1082 
1083 	set_crypto_boot_test_finished();
1084 
1085 	for (;;) {
1086 		struct crypto_larval *larval = NULL;
1087 		struct crypto_alg *q;
1088 
1089 		down_write(&crypto_alg_sem);
1090 
1091 		list_for_each_entry(q, &crypto_alg_list, cra_list) {
1092 			struct crypto_larval *l;
1093 
1094 			if (!crypto_is_larval(q))
1095 				continue;
1096 
1097 			l = (void *)q;
1098 
1099 			if (!crypto_is_test_larval(l))
1100 				continue;
1101 
1102 			if (l->test_started)
1103 				continue;
1104 
1105 			l->test_started = true;
1106 			larval = l;
1107 			break;
1108 		}
1109 
1110 		up_write(&crypto_alg_sem);
1111 
1112 		if (!larval)
1113 			break;
1114 
1115 		crypto_schedule_test(larval);
1116 	}
1117 }
1118 
1119 static int __init crypto_algapi_init(void)
1120 {
1121 	crypto_init_proc();
1122 	crypto_start_tests();
1123 	return 0;
1124 }
1125 
1126 static void __exit crypto_algapi_exit(void)
1127 {
1128 	crypto_exit_proc();
1129 }
1130 
1131 /*
1132  * We run this at late_initcall so that all the built-in algorithms
1133  * have had a chance to register themselves first.
1134  */
1135 late_initcall(crypto_algapi_init);
1136 module_exit(crypto_algapi_exit);
1137 
1138 MODULE_LICENSE("GPL");
1139 MODULE_DESCRIPTION("Cryptographic algorithms API");
1140 MODULE_SOFTDEP("pre: cryptomgr");
1141