32 static int ahash_def_finup(struct ahash_request *req);
102 int crypto_hash_walk_first(struct ahash_request *req,  in crypto_hash_walk_first()  argument
105 	walk->total = req->nbytes;  in crypto_hash_walk_first()
111 	walk->flags = req->base.flags;  in crypto_hash_walk_first()
113 	if (ahash_request_isvirt(req)) {  in crypto_hash_walk_first()
114 		walk->data = req->svirt;  in crypto_hash_walk_first()
116 		return req->nbytes;  in crypto_hash_walk_first()
119 	walk->sg = req->src;  in crypto_hash_walk_first()
162 static inline struct shash_desc *prepare_shash_desc(struct ahash_request *req,  in prepare_shash_desc()  argument
165 	struct shash_desc *desc = ahash_request_ctx(req);  in prepare_shash_desc()
171 int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc)  in shash_ahash_update()  argument
176 	for (nbytes = crypto_hash_walk_first(req, &walk); nbytes > 0;  in shash_ahash_update()
184 int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc)  in shash_ahash_finup()  argument
189 	nbytes = crypto_hash_walk_first(req, &walk);  in shash_ahash_finup()
191 		return crypto_shash_final(desc, req->result);  in shash_ahash_finup()
196 					    req->result) :  in shash_ahash_finup()
205 int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc)  in shash_ahash_digest()  argument
207 	unsigned int nbytes = req->nbytes;  in shash_ahash_digest()
214 	data = req->svirt;  in shash_ahash_digest()
215 	if (!nbytes || ahash_request_isvirt(req))  in shash_ahash_digest()
216 		return crypto_shash_digest(desc, data, nbytes, req->result);  in shash_ahash_digest()
218 	sg = req->src;  in shash_ahash_digest()
221 		       shash_ahash_finup(req, desc);  in shash_ahash_digest()
227 		return crypto_shash_digest(desc, data, nbytes, req->result);  in shash_ahash_digest()
234 		       shash_ahash_finup(req, desc);  in shash_ahash_digest()
238 				  req->result);  in shash_ahash_digest()
322 static int ahash_do_req_chain(struct ahash_request *req,  in ahash_do_req_chain()  argument
323 			      int (*const *op)(struct ahash_request *req))  in ahash_do_req_chain()  argument
325 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in ahash_do_req_chain()
328 	if (crypto_ahash_req_virt(tfm) || !ahash_request_isvirt(req))  in ahash_do_req_chain()
329 		return (*op)(req);  in ahash_do_req_chain()
344 			ahash_request_set_tfm(req, crypto_ahash_fb(tfm));  in ahash_do_req_chain()
345 			err = crypto_ahash_digest(req);  in ahash_do_req_chain()
349 		err = crypto_ahash_export(req, state);  in ahash_do_req_chain()
350 		ahash_request_set_tfm(req, crypto_ahash_fb(tfm));  in ahash_do_req_chain()
351 		err = err ?: crypto_ahash_import(req, state);  in ahash_do_req_chain()
354 			err = err ?: crypto_ahash_finup(req);  in ahash_do_req_chain()
359 		      crypto_ahash_update(req) ?:  in ahash_do_req_chain()
360 		      crypto_ahash_export(req, state);  in ahash_do_req_chain()
362 		ahash_request_set_tfm(req, tfm);  in ahash_do_req_chain()
363 		return err ?: crypto_ahash_import(req, state);  in ahash_do_req_chain()
366 		ahash_request_set_tfm(req, tfm);  in ahash_do_req_chain()
371 int crypto_ahash_init(struct ahash_request *req)  in crypto_ahash_init()  argument
373 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_init()
376 		return crypto_shash_init(prepare_shash_desc(req, tfm));  in crypto_ahash_init()
379 	if (ahash_req_on_stack(req) && ahash_is_async(tfm))  in crypto_ahash_init()
382 		u8 *buf = ahash_request_ctx(req);  in crypto_ahash_init()
387 	return crypto_ahash_alg(tfm)->init(req);  in crypto_ahash_init()
391 static void ahash_save_req(struct ahash_request *req, crypto_completion_t cplt)  in ahash_save_req()  argument
393 	req->saved_complete = req->base.complete;  in ahash_save_req()
394 	req->saved_data = req->base.data;  in ahash_save_req()
395 	req->base.complete = cplt;  in ahash_save_req()
396 	req->base.data = req;  in ahash_save_req()
399 static void ahash_restore_req(struct ahash_request *req)  in ahash_restore_req()  argument
401 	req->base.complete = req->saved_complete;  in ahash_restore_req()
402 	req->base.data = req->saved_data;  in ahash_restore_req()
405 static int ahash_update_finish(struct ahash_request *req, int err)  in ahash_update_finish()  argument
407 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in ahash_update_finish()
410 	u8 *blenp = ahash_request_ctx(req);  in ahash_update_finish()
419 		req->src = req->sg_head + 1;  in ahash_update_finish()
420 		if (sg_is_chain(req->src))  in ahash_update_finish()
421 			req->src = sg_chain_ptr(req->src);  in ahash_update_finish()
424 	req->nbytes += nonzero - blen;  in ahash_update_finish()
431 	if (ahash_request_isvirt(req))  in ahash_update_finish()
432 		memcpy(buf, req->svirt + req->nbytes - blen, blen);  in ahash_update_finish()
434 		memcpy_from_sglist(buf, req->src, req->nbytes - blen, blen);  in ahash_update_finish()
437 	ahash_restore_req(req);  in ahash_update_finish()
447 int crypto_ahash_update(struct ahash_request *req)  in crypto_ahash_update()  argument
449 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_update()
452 	u8 *blenp = ahash_request_ctx(req);  in crypto_ahash_update()
457 		return shash_ahash_update(req, ahash_request_ctx(req));  in crypto_ahash_update()
458 	if (ahash_req_on_stack(req) && ahash_is_async(tfm))  in crypto_ahash_update()
461 		return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->update);  in crypto_ahash_update()
467 	if (blen + req->nbytes < bs + nonzero) {  in crypto_ahash_update()
468 		if (ahash_request_isvirt(req))  in crypto_ahash_update()
469 			memcpy(buf + blen, req->svirt, req->nbytes);  in crypto_ahash_update()
471 			memcpy_from_sglist(buf + blen, req->src, 0,  in crypto_ahash_update()
472 					   req->nbytes);  in crypto_ahash_update()
474 		*blenp += req->nbytes;  in crypto_ahash_update()
479 		memset(req->sg_head, 0, sizeof(req->sg_head[0]));  in crypto_ahash_update()
480 		sg_set_buf(req->sg_head, buf, blen);  in crypto_ahash_update()
481 		if (req->src != req->sg_head + 1)  in crypto_ahash_update()
482 			sg_chain(req->sg_head, 2, req->src);  in crypto_ahash_update()
483 		req->src = req->sg_head;  in crypto_ahash_update()
484 		req->nbytes += blen;  in crypto_ahash_update()
486 	req->nbytes -= nonzero;  in crypto_ahash_update()
488 	ahash_save_req(req, ahash_update_done);  in crypto_ahash_update()
490 	err = ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->update);  in crypto_ahash_update()
494 	return ahash_update_finish(req, err);  in crypto_ahash_update()
498 static int ahash_finup_finish(struct ahash_request *req, int err)  in ahash_finup_finish()  argument
500 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in ahash_finup_finish()
501 	u8 *blenp = ahash_request_ctx(req);  in ahash_finup_finish()
508 		if (sg_is_last(req->src))  in ahash_finup_finish()
509 			req->src = NULL;  in ahash_finup_finish()
511 			req->src = req->sg_head + 1;  in ahash_finup_finish()
512 			if (sg_is_chain(req->src))  in ahash_finup_finish()
513 				req->src = sg_chain_ptr(req->src);  in ahash_finup_finish()
515 		req->nbytes -= blen;  in ahash_finup_finish()
518 	ahash_restore_req(req);  in ahash_finup_finish()
528 int crypto_ahash_finup(struct ahash_request *req)  in crypto_ahash_finup()  argument
530 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_finup()
532 	u8 *blenp = ahash_request_ctx(req);  in crypto_ahash_finup()
537 		return shash_ahash_finup(req, ahash_request_ctx(req));  in crypto_ahash_finup()
538 	if (ahash_req_on_stack(req) && ahash_is_async(tfm))  in crypto_ahash_finup()
541 		return ahash_def_finup(req);  in crypto_ahash_finup()
543 		return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->finup);  in crypto_ahash_finup()
550 		memset(req->sg_head, 0, sizeof(req->sg_head[0]));  in crypto_ahash_finup()
551 		sg_set_buf(req->sg_head, buf, blen);  in crypto_ahash_finup()
552 		if (!req->src)  in crypto_ahash_finup()
553 			sg_mark_end(req->sg_head);  in crypto_ahash_finup()
554 		else if (req->src != req->sg_head + 1)  in crypto_ahash_finup()
555 			sg_chain(req->sg_head, 2, req->src);  in crypto_ahash_finup()
556 		req->src = req->sg_head;  in crypto_ahash_finup()
557 		req->nbytes += blen;  in crypto_ahash_finup()
560 	ahash_save_req(req, ahash_finup_done);  in crypto_ahash_finup()
562 	err = ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->finup);  in crypto_ahash_finup()
566 	return ahash_finup_finish(req, err);  in crypto_ahash_finup()
570 int crypto_ahash_digest(struct ahash_request *req)  in crypto_ahash_digest()  argument
572 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_digest()
575 		return shash_ahash_digest(req, prepare_shash_desc(req, tfm));  in crypto_ahash_digest()
576 	if (ahash_req_on_stack(req) && ahash_is_async(tfm))  in crypto_ahash_digest()
580 	return ahash_do_req_chain(req, &crypto_ahash_alg(tfm)->digest);  in crypto_ahash_digest()
595 static int ahash_def_finup_finish1(struct ahash_request *req, int err)  in ahash_def_finup_finish1()  argument
597 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in ahash_def_finup_finish1()
602 	req->base.complete = ahash_def_finup_done2;  in ahash_def_finup_finish1()
604 	err = crypto_ahash_alg(tfm)->final(req);  in ahash_def_finup_finish1()
609 	ahash_restore_req(req);  in ahash_def_finup_finish1()
618 static int ahash_def_finup(struct ahash_request *req)  in ahash_def_finup()  argument
622 	ahash_save_req(req, ahash_def_finup_done1);  in ahash_def_finup()
624 	err = crypto_ahash_update(req);  in ahash_def_finup()
628 	return ahash_def_finup_finish1(req, err);  in ahash_def_finup()
631 int crypto_ahash_export_core(struct ahash_request *req, void *out)  in crypto_ahash_export_core()  argument
633 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_export_core()
636 		return crypto_shash_export_core(ahash_request_ctx(req), out);  in crypto_ahash_export_core()
637 	return crypto_ahash_alg(tfm)->export_core(req, out);  in crypto_ahash_export_core()
641 int crypto_ahash_export(struct ahash_request *req, void *out)  in crypto_ahash_export()  argument
643 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_export()
646 		return crypto_shash_export(ahash_request_ctx(req), out);  in crypto_ahash_export()
651 		u8 *buf = ahash_request_ctx(req);  in crypto_ahash_export()
655 	return crypto_ahash_alg(tfm)->export(req, out);  in crypto_ahash_export()
659 int crypto_ahash_import_core(struct ahash_request *req, const void *in)  in crypto_ahash_import_core()  argument
661 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_import_core()
664 		return crypto_shash_import_core(prepare_shash_desc(req, tfm),  in crypto_ahash_import_core()
670 		u8 *buf = ahash_request_ctx(req);  in crypto_ahash_import_core()
674 	return crypto_ahash_alg(tfm)->import_core(req, in);  in crypto_ahash_import_core()
678 int crypto_ahash_import(struct ahash_request *req, const void *in)  in crypto_ahash_import()  argument
680 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in crypto_ahash_import()
683 		return crypto_shash_import(prepare_shash_desc(req, tfm), in);  in crypto_ahash_import()
690 		u8 *buf = ahash_request_ctx(req);  in crypto_ahash_import()
696 	return crypto_ahash_alg(tfm)->import(req, in);  in crypto_ahash_import()
936 static int ahash_default_export_core(struct ahash_request *req, void *out)  in ahash_default_export_core()  argument
941 static int ahash_default_import_core(struct ahash_request *req, const void *in)  in ahash_default_import_core()  argument
1062 void ahash_request_free(struct ahash_request *req)  in ahash_request_free()  argument
1064 	if (unlikely(!req))  in ahash_request_free()
1067 	if (!ahash_req_on_stack(req)) {  in ahash_request_free()
1068 		kfree(req);  in ahash_request_free()
1072 	ahash_request_zero(req);  in ahash_request_free()
1079 	HASH_REQUEST_ON_STACK(req, crypto_ahash_fb(tfm));  in crypto_hash_digest()
1082 	ahash_request_set_callback(req, 0, NULL, NULL);  in crypto_hash_digest()
1083 	ahash_request_set_virt(req, data, out, len);  in crypto_hash_digest()
1084 	err = crypto_ahash_digest(req);  in crypto_hash_digest()
1086 	ahash_request_zero(req);  in crypto_hash_digest()