32 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])  in poly1305_init_arch()  argument
34 	poly1305_init_arm(&dctx->h, key);  in poly1305_init_arch()
35 	dctx->s[0] = get_unaligned_le32(key + 16);  in poly1305_init_arch()
36 	dctx->s[1] = get_unaligned_le32(key + 20);  in poly1305_init_arch()
37 	dctx->s[2] = get_unaligned_le32(key + 24);  in poly1305_init_arch()
38 	dctx->s[3] = get_unaligned_le32(key + 28);  in poly1305_init_arch()
39 	dctx->buflen = 0;  in poly1305_init_arch()
45 	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);  in arm_poly1305_init()  local
47 	dctx->buflen = 0;  in arm_poly1305_init()
48 	dctx->rset = 0;  in arm_poly1305_init()
49 	dctx->sset = false;  in arm_poly1305_init()
54 static void arm_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src,  in arm_poly1305_blocks()  argument
57 	if (unlikely(!dctx->sset)) {  in arm_poly1305_blocks()
58 		if (!dctx->rset) {  in arm_poly1305_blocks()
59 			poly1305_init_arm(&dctx->h, src);  in arm_poly1305_blocks()
62 			dctx->rset = 1;  in arm_poly1305_blocks()
65 			dctx->s[0] = get_unaligned_le32(src +  0);  in arm_poly1305_blocks()
66 			dctx->s[1] = get_unaligned_le32(src +  4);  in arm_poly1305_blocks()
67 			dctx->s[2] = get_unaligned_le32(src +  8);  in arm_poly1305_blocks()
68 			dctx->s[3] = get_unaligned_le32(src + 12);  in arm_poly1305_blocks()
71 			dctx->sset = true;  in arm_poly1305_blocks()
80 		poly1305_blocks_neon(&dctx->h, src, len, hibit);  in arm_poly1305_blocks()
82 		poly1305_blocks_arm(&dctx->h, src, len, hibit);  in arm_poly1305_blocks()
85 static void arm_poly1305_do_update(struct poly1305_desc_ctx *dctx,  in arm_poly1305_do_update()  argument
88 	if (unlikely(dctx->buflen)) {  in arm_poly1305_do_update()
89 		u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen);  in arm_poly1305_do_update()
91 		memcpy(dctx->buf + dctx->buflen, src, bytes);  in arm_poly1305_do_update()
94 		dctx->buflen += bytes;  in arm_poly1305_do_update()
96 		if (dctx->buflen == POLY1305_BLOCK_SIZE) {  in arm_poly1305_do_update()
97 			arm_poly1305_blocks(dctx, dctx->buf,  in arm_poly1305_do_update()
99 			dctx->buflen = 0;  in arm_poly1305_do_update()
104 		arm_poly1305_blocks(dctx, src, len, 1, do_neon);  in arm_poly1305_do_update()
110 		dctx->buflen = len;  in arm_poly1305_do_update()
111 		memcpy(dctx->buf, src, len);  in arm_poly1305_do_update()
118 	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);  in arm_poly1305_update()  local
120 	arm_poly1305_do_update(dctx, src, srclen, false);  in arm_poly1305_update()
128 	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);  in arm_poly1305_update_neon()  local
133 	arm_poly1305_do_update(dctx, src, srclen, do_neon);  in arm_poly1305_update_neon()
139 void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,  in poly1305_update_arch()  argument
145 	if (unlikely(dctx->buflen)) {  in poly1305_update_arch()
146 		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);  in poly1305_update_arch()
148 		memcpy(dctx->buf + dctx->buflen, src, bytes);  in poly1305_update_arch()
151 		dctx->buflen += bytes;  in poly1305_update_arch()
153 		if (dctx->buflen == POLY1305_BLOCK_SIZE) {  in poly1305_update_arch()
154 			poly1305_blocks_arm(&dctx->h, dctx->buf,  in poly1305_update_arch()
156 			dctx->buflen = 0;  in poly1305_update_arch()
168 				poly1305_blocks_neon(&dctx->h, src, todo, 1);  in poly1305_update_arch()
175 			poly1305_blocks_arm(&dctx->h, src, len, 1);  in poly1305_update_arch()
182 		dctx->buflen = nbytes;  in poly1305_update_arch()
183 		memcpy(dctx->buf, src, nbytes);  in poly1305_update_arch()
188 void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)  in poly1305_final_arch()  argument
190 	if (unlikely(dctx->buflen)) {  in poly1305_final_arch()
191 		dctx->buf[dctx->buflen++] = 1;  in poly1305_final_arch()
192 		memset(dctx->buf + dctx->buflen, 0,  in poly1305_final_arch()
193 		       POLY1305_BLOCK_SIZE - dctx->buflen);  in poly1305_final_arch()
194 		poly1305_blocks_arm(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);  in poly1305_final_arch()
197 	poly1305_emit_arm(&dctx->h, dst, dctx->s);  in poly1305_final_arch()
198 	*dctx = (struct poly1305_desc_ctx){};  in poly1305_final_arch()
204 	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);  in arm_poly1305_final()  local
206 	if (unlikely(!dctx->sset))  in arm_poly1305_final()
209 	poly1305_final_arch(dctx, dst);  in arm_poly1305_final()