xref: /linux/arch/arm64/crypto/sha2-ce-core.S (revision bfd5bb6f90af092aa345b15cd78143956a13c2a8)
1/*
2 * sha2-ce-core.S - core SHA-224/SHA-256 transform using v8 Crypto Extensions
3 *
4 * Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/linkage.h>
12#include <asm/assembler.h>
13
14	.text
15	.arch		armv8-a+crypto
16
17	dga		.req	q20
18	dgav		.req	v20
19	dgb		.req	q21
20	dgbv		.req	v21
21
22	t0		.req	v22
23	t1		.req	v23
24
25	dg0q		.req	q24
26	dg0v		.req	v24
27	dg1q		.req	q25
28	dg1v		.req	v25
29	dg2q		.req	q26
30	dg2v		.req	v26
31
32	.macro		add_only, ev, rc, s0
33	mov		dg2v.16b, dg0v.16b
34	.ifeq		\ev
35	add		t1.4s, v\s0\().4s, \rc\().4s
36	sha256h		dg0q, dg1q, t0.4s
37	sha256h2	dg1q, dg2q, t0.4s
38	.else
39	.ifnb		\s0
40	add		t0.4s, v\s0\().4s, \rc\().4s
41	.endif
42	sha256h		dg0q, dg1q, t1.4s
43	sha256h2	dg1q, dg2q, t1.4s
44	.endif
45	.endm
46
47	.macro		add_update, ev, rc, s0, s1, s2, s3
48	sha256su0	v\s0\().4s, v\s1\().4s
49	add_only	\ev, \rc, \s1
50	sha256su1	v\s0\().4s, v\s2\().4s, v\s3\().4s
51	.endm
52
53	/*
54	 * The SHA-256 round constants
55	 */
56	.section	".rodata", "a"
57	.align		4
58.Lsha2_rcon:
59	.word		0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
60	.word		0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
61	.word		0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
62	.word		0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
63	.word		0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
64	.word		0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
65	.word		0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
66	.word		0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
67	.word		0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
68	.word		0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
69	.word		0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
70	.word		0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
71	.word		0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
72	.word		0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
73	.word		0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
74	.word		0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
75
76	/*
77	 * void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src,
78	 *			  int blocks)
79	 */
80	.text
81ENTRY(sha2_ce_transform)
82	frame_push	3
83
84	mov		x19, x0
85	mov		x20, x1
86	mov		x21, x2
87
88	/* load round constants */
890:	adr_l		x8, .Lsha2_rcon
90	ld1		{ v0.4s- v3.4s}, [x8], #64
91	ld1		{ v4.4s- v7.4s}, [x8], #64
92	ld1		{ v8.4s-v11.4s}, [x8], #64
93	ld1		{v12.4s-v15.4s}, [x8]
94
95	/* load state */
96	ld1		{dgav.4s, dgbv.4s}, [x19]
97
98	/* load sha256_ce_state::finalize */
99	ldr_l		w4, sha256_ce_offsetof_finalize, x4
100	ldr		w4, [x19, x4]
101
102	/* load input */
1031:	ld1		{v16.4s-v19.4s}, [x20], #64
104	sub		w21, w21, #1
105
106CPU_LE(	rev32		v16.16b, v16.16b	)
107CPU_LE(	rev32		v17.16b, v17.16b	)
108CPU_LE(	rev32		v18.16b, v18.16b	)
109CPU_LE(	rev32		v19.16b, v19.16b	)
110
1112:	add		t0.4s, v16.4s, v0.4s
112	mov		dg0v.16b, dgav.16b
113	mov		dg1v.16b, dgbv.16b
114
115	add_update	0,  v1, 16, 17, 18, 19
116	add_update	1,  v2, 17, 18, 19, 16
117	add_update	0,  v3, 18, 19, 16, 17
118	add_update	1,  v4, 19, 16, 17, 18
119
120	add_update	0,  v5, 16, 17, 18, 19
121	add_update	1,  v6, 17, 18, 19, 16
122	add_update	0,  v7, 18, 19, 16, 17
123	add_update	1,  v8, 19, 16, 17, 18
124
125	add_update	0,  v9, 16, 17, 18, 19
126	add_update	1, v10, 17, 18, 19, 16
127	add_update	0, v11, 18, 19, 16, 17
128	add_update	1, v12, 19, 16, 17, 18
129
130	add_only	0, v13, 17
131	add_only	1, v14, 18
132	add_only	0, v15, 19
133	add_only	1
134
135	/* update state */
136	add		dgav.4s, dgav.4s, dg0v.4s
137	add		dgbv.4s, dgbv.4s, dg1v.4s
138
139	/* handled all input blocks? */
140	cbz		w21, 3f
141
142	if_will_cond_yield_neon
143	st1		{dgav.4s, dgbv.4s}, [x19]
144	do_cond_yield_neon
145	b		0b
146	endif_yield_neon
147
148	b		1b
149
150	/*
151	 * Final block: add padding and total bit count.
152	 * Skip if the input size was not a round multiple of the block size,
153	 * the padding is handled by the C code in that case.
154	 */
1553:	cbz		x4, 4f
156	ldr_l		w4, sha256_ce_offsetof_count, x4
157	ldr		x4, [x19, x4]
158	movi		v17.2d, #0
159	mov		x8, #0x80000000
160	movi		v18.2d, #0
161	ror		x7, x4, #29		// ror(lsl(x4, 3), 32)
162	fmov		d16, x8
163	mov		x4, #0
164	mov		v19.d[0], xzr
165	mov		v19.d[1], x7
166	b		2b
167
168	/* store new state */
1694:	st1		{dgav.4s, dgbv.4s}, [x19]
170	frame_pop
171	ret
172ENDPROC(sha2_ce_transform)
173