32 static int ahash_def_finup(struct ahash_request *req);
102 int crypto_hash_walk_first(struct ahash_request *req,  in crypto_hash_walk_first()  argument
105 	walk->total = req->nbytes;  in crypto_hash_walk_first()
111 	walk->flags = req->base.flags;  in crypto_hash_walk_first()
113 	if (ahash_request_isvirt(req)) {  in crypto_hash_walk_first()
114 		walk->data = req->svirt;  in crypto_hash_walk_first()
116 		return req->nbytes;  in crypto_hash_walk_first()
119 	walk->sg = req->src;  in crypto_hash_walk_first()
162 static inline struct shash_desc *prepare_shash_desc(struct ahash_request *req,  in prepare_shash_desc()  argument
165 	struct shash_desc *desc = ahash_request_ctx(req);  in prepare_shash_desc()
171 int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc)  in shash_ahash_update()  argument
176 	for (nbytes = crypto_hash_walk_first(req, &walk); nbytes > 0;  in shash_ahash_update()
184 int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc)  in shash_ahash_finup()  argument
189 	nbytes = crypto_hash_walk_first(req, &walk);  in shash_ahash_finup()
191 		return crypto_shash_final(desc, req->result);  in shash_ahash_finup()
196 					    req->result) :  in shash_ahash_finup()
205 int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc)  in shash_ahash_digest()  argument
207 	unsigned int nbytes = req->nbytes;  in shash_ahash_digest()
214 	data = req->svirt;  in shash_ahash_digest()
215 	if (!nbytes || ahash_request_isvirt(req))  in shash_ahash_digest()
216 		return crypto_shash_digest(desc, data, nbytes, req->result);  in shash_ahash_digest()
218 	sg = req->src;  in shash_ahash_digest()
221 		       shash_ahash_finup(req, desc);  in shash_ahash_digest()
227 		return crypto_shash_digest(desc, data, nbytes, req->result);  in shash_ahash_digest()
234 		       shash_ahash_finup(req, desc);  in shash_ahash_digest()
238 				  req->result);  in shash_ahash_digest()
322 static int ahash_do_req_chain(struct ahash_request *req,  in ahash_do_req_chain()  argument
323 			      int (*const *op)(struct ahash_request *req))  in ahash_do_req_chain()  argument
325 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in ahash_do_req_chain()
328 	if (crypto_ahash_req_virt(tfm) || !ahash_request_isvirt(req))  in ahash_do_req_chain()
329 		return (*op)(req);  in ahash_do_req_chain()
344 			ahash_request_set_tfm(req, crypto_ahash_fb(tfm));  in ahash_do_req_chain()
345 			err = crypto_ahash_digest(req);  in ahash_do_req_chain()
349 		err = crypto_ahash_export(req, state);  in ahash_do_req_chain()
350 		ahash_request_set_tfm(req, crypto_ahash_fb(tfm));  in ahash_do_req_chain()
351 		err = err ?: crypto_ahash_import(req, state);  in ahash_do_req_chain()
354 			err = err ?: crypto_ahash_finup(req);  in ahash_do_req_chain()
359 		      crypto_ahash_update(req) ?:  in ahash_do_req_chain()
360 		      crypto_ahash_export(req, state);  in ahash_do_req_chain()
362 		ahash_request_set_tfm(req, tfm);  in ahash_do_req_chain()
363 		return err ?: crypto_ahash_import(req, state);  in ahash_do_req_chain()
366 		ahash_request_set_tfm(req, tfm);  in ahash_do_req_chain()
371 int crypto_ahash_init(struct ahash_request *req)  in crypto_ahash_init()  argument
373 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_init()
376 		return crypto_shash_init(prepare_shash_desc(req, tfm));  in crypto_ahash_init()
379 	if (ahash_req_on_stack(req) && ahash_is_async(tfm))  in crypto_ahash_init()
382 		u8 *buf = ahash_request_ctx(req);  in crypto_ahash_init()
387 	return crypto_ahash_alg(tfm)->init(req);  in crypto_ahash_init()
391 static void ahash_save_req(struct ahash_request *req, crypto_completion_t cplt)  in ahash_save_req()  argument
393 	req->saved_complete = req->base.complete;  in ahash_save_req()
394 	req->saved_data = req->base.data;  in ahash_save_req()
395 	req->base.complete = cplt;  in ahash_save_req()
396 	req->base.data = req;  in ahash_save_req()
399 static void ahash_restore_req(struct ahash_request *req)  in ahash_restore_req()  argument
401 	req->base.complete = req->saved_complete;  in ahash_restore_req()
402 	req->base.data = req->saved_data;  in ahash_restore_req()
405 static int ahash_update_finish(struct ahash_request *req, int err)  in ahash_update_finish()  argument
407 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in ahash_update_finish()
410 	u8 *blenp = ahash_request_ctx(req);  in ahash_update_finish()
419 		req->src = req->sg_head + 1;  in ahash_update_finish()
420 		if (sg_is_chain(req->src))  in ahash_update_finish()
421 			req->src = sg_chain_ptr(req->src);  in ahash_update_finish()
424 	req->nbytes += nonzero - blen;  in ahash_update_finish()
427 	if (ahash_request_isvirt(req))  in ahash_update_finish()
428 		memcpy(buf, req->svirt + req->nbytes - blen, blen);  in ahash_update_finish()
430 		memcpy_from_sglist(buf, req->src, req->nbytes - blen, blen);  in ahash_update_finish()
433 	ahash_restore_req(req);  in ahash_update_finish()
443 int crypto_ahash_update(struct ahash_request *req)  in crypto_ahash_update()  argument
445 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_update()
448 	u8 *blenp = ahash_request_ctx(req);  in crypto_ahash_update()
453 		return shash_ahash_update(req, ahash_request_ctx(req));  in crypto_ahash_update()
454 	if (ahash_req_on_stack(req) && ahash_is_async(tfm))  in crypto_ahash_update()
457 		return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->update);  in crypto_ahash_update()
463 	if (blen + req->nbytes < bs + nonzero) {  in crypto_ahash_update()
464 		if (ahash_request_isvirt(req))  in crypto_ahash_update()
465 			memcpy(buf + blen, req->svirt, req->nbytes);  in crypto_ahash_update()
467 			memcpy_from_sglist(buf + blen, req->src, 0,  in crypto_ahash_update()
468 					   req->nbytes);  in crypto_ahash_update()
470 		*blenp += req->nbytes;  in crypto_ahash_update()
475 		memset(req->sg_head, 0, sizeof(req->sg_head[0]));  in crypto_ahash_update()
476 		sg_set_buf(req->sg_head, buf, blen);  in crypto_ahash_update()
477 		if (req->src != req->sg_head + 1)  in crypto_ahash_update()
478 			sg_chain(req->sg_head, 2, req->src);  in crypto_ahash_update()
479 		req->src = req->sg_head;  in crypto_ahash_update()
480 		req->nbytes += blen;  in crypto_ahash_update()
482 	req->nbytes -= nonzero;  in crypto_ahash_update()
484 	ahash_save_req(req, ahash_update_done);  in crypto_ahash_update()
486 	err = ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->update);  in crypto_ahash_update()
490 	return ahash_update_finish(req, err);  in crypto_ahash_update()
494 static int ahash_finup_finish(struct ahash_request *req, int err)  in ahash_finup_finish()  argument
496 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in ahash_finup_finish()
497 	u8 *blenp = ahash_request_ctx(req);  in ahash_finup_finish()
504 		if (sg_is_last(req->src))  in ahash_finup_finish()
505 			req->src = NULL;  in ahash_finup_finish()
507 			req->src = req->sg_head + 1;  in ahash_finup_finish()
508 			if (sg_is_chain(req->src))  in ahash_finup_finish()
509 				req->src = sg_chain_ptr(req->src);  in ahash_finup_finish()
511 		req->nbytes -= blen;  in ahash_finup_finish()
514 	ahash_restore_req(req);  in ahash_finup_finish()
524 int crypto_ahash_finup(struct ahash_request *req)  in crypto_ahash_finup()  argument
526 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_finup()
528 	u8 *blenp = ahash_request_ctx(req);  in crypto_ahash_finup()
533 		return shash_ahash_finup(req, ahash_request_ctx(req));  in crypto_ahash_finup()
534 	if (ahash_req_on_stack(req) && ahash_is_async(tfm))  in crypto_ahash_finup()
537 		return ahash_def_finup(req);  in crypto_ahash_finup()
539 		return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->finup);  in crypto_ahash_finup()
546 		memset(req->sg_head, 0, sizeof(req->sg_head[0]));  in crypto_ahash_finup()
547 		sg_set_buf(req->sg_head, buf, blen);  in crypto_ahash_finup()
548 		if (!req->src)  in crypto_ahash_finup()
549 			sg_mark_end(req->sg_head);  in crypto_ahash_finup()
550 		else if (req->src != req->sg_head + 1)  in crypto_ahash_finup()
551 			sg_chain(req->sg_head, 2, req->src);  in crypto_ahash_finup()
552 		req->src = req->sg_head;  in crypto_ahash_finup()
553 		req->nbytes += blen;  in crypto_ahash_finup()
556 	ahash_save_req(req, ahash_finup_done);  in crypto_ahash_finup()
558 	err = ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->finup);  in crypto_ahash_finup()
562 	return ahash_finup_finish(req, err);  in crypto_ahash_finup()
566 int crypto_ahash_digest(struct ahash_request *req)  in crypto_ahash_digest()  argument
568 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_digest()
571 		return shash_ahash_digest(req, prepare_shash_desc(req, tfm));  in crypto_ahash_digest()
572 	if (ahash_req_on_stack(req) && ahash_is_async(tfm))  in crypto_ahash_digest()
576 	return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->digest);  in crypto_ahash_digest()
591 static int ahash_def_finup_finish1(struct ahash_request *req, int err)  in ahash_def_finup_finish1()  argument
593 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in ahash_def_finup_finish1()
598 	req->base.complete = ahash_def_finup_done2;  in ahash_def_finup_finish1()
600 	err = crypto_ahash_alg(tfm)->final(req);  in ahash_def_finup_finish1()
605 	ahash_restore_req(req);  in ahash_def_finup_finish1()
614 static int ahash_def_finup(struct ahash_request *req)  in ahash_def_finup()  argument
618 	ahash_save_req(req, ahash_def_finup_done1);  in ahash_def_finup()
620 	err = crypto_ahash_update(req);  in ahash_def_finup()
624 	return ahash_def_finup_finish1(req, err);  in ahash_def_finup()
627 int crypto_ahash_export_core(struct ahash_request *req, void *out)  in crypto_ahash_export_core()  argument
629 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_export_core()
632 		return crypto_shash_export_core(ahash_request_ctx(req), out);  in crypto_ahash_export_core()
633 	return crypto_ahash_alg(tfm)->export_core(req, out);  in crypto_ahash_export_core()
637 int crypto_ahash_export(struct ahash_request *req, void *out)  in crypto_ahash_export()  argument
639 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_export()
642 		return crypto_shash_export(ahash_request_ctx(req), out);  in crypto_ahash_export()
647 		u8 *buf = ahash_request_ctx(req);  in crypto_ahash_export()
651 	return crypto_ahash_alg(tfm)->export(req, out);  in crypto_ahash_export()
655 int crypto_ahash_import_core(struct ahash_request *req, const void *in)  in crypto_ahash_import_core()  argument
657 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_import_core()
660 		return crypto_shash_import_core(prepare_shash_desc(req, tfm),  in crypto_ahash_import_core()
664 	return crypto_ahash_alg(tfm)->import_core(req, in);  in crypto_ahash_import_core()
668 int crypto_ahash_import(struct ahash_request *req, const void *in)  in crypto_ahash_import()  argument
670 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_import()
673 		return crypto_shash_import(prepare_shash_desc(req, tfm), in);  in crypto_ahash_import()
678 		u8 *buf = ahash_request_ctx(req);  in crypto_ahash_import()
682 	return crypto_ahash_alg(tfm)->import(req, in);  in crypto_ahash_import()
922 static int ahash_default_export_core(struct ahash_request *req, void *out)  in ahash_default_export_core()  argument
927 static int ahash_default_import_core(struct ahash_request *req, const void *in)  in ahash_default_import_core()  argument
1048 void ahash_request_free(struct ahash_request *req)  in ahash_request_free()  argument
1050 	if (unlikely(!req))  in ahash_request_free()
1053 	if (!ahash_req_on_stack(req)) {  in ahash_request_free()
1054 		kfree(req);  in ahash_request_free()
1058 	ahash_request_zero(req);  in ahash_request_free()
1065 	HASH_REQUEST_ON_STACK(req, crypto_ahash_fb(tfm));  in crypto_hash_digest()
1068 	ahash_request_set_callback(req, 0, NULL, NULL);  in crypto_hash_digest()
1069 	ahash_request_set_virt(req, data, out, len);  in crypto_hash_digest()
1070 	err = crypto_ahash_digest(req);  in crypto_hash_digest()
1072 	ahash_request_zero(req);  in crypto_hash_digest()