xref: /linux/arch/arm64/crypto/aes-ce.S (revision 55d0969c451159cff86949b38c39171cab962069)
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with
4 *                                    Crypto Extensions
5 *
6 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
7 */
8
9#include <linux/linkage.h>
10#include <asm/assembler.h>
11
12#define AES_FUNC_START(func)		SYM_FUNC_START(ce_ ## func)
13#define AES_FUNC_END(func)		SYM_FUNC_END(ce_ ## func)
14
15	.arch		armv8-a+crypto
16
17	xtsmask		.req	v16
18	cbciv		.req	v16
19	vctr		.req	v16
20
21	.macro		xts_reload_mask, tmp
22	.endm
23
24	.macro		xts_cts_skip_tw, reg, lbl
25	.endm
26
27	/* preload all round keys */
28	.macro		load_round_keys, rk, nr, tmp
29	add		\tmp, \rk, \nr, sxtw #4
30	sub		\tmp, \tmp, #160
31	ld1		{v17.4s-v20.4s}, [\rk]
32	ld1		{v21.4s-v24.4s}, [\tmp], #64
33	ld1		{v25.4s-v28.4s}, [\tmp], #64
34	ld1		{v29.4s-v31.4s}, [\tmp]
35	.endm
36
37	/* prepare for encryption with key in rk[] */
38	.macro		enc_prepare, rounds, rk, temp
39	load_round_keys	\rk, \rounds, \temp
40	.endm
41
42	/* prepare for encryption (again) but with new key in rk[] */
43	.macro		enc_switch_key, rounds, rk, temp
44	load_round_keys	\rk, \rounds, \temp
45	.endm
46
47	/* prepare for decryption with key in rk[] */
48	.macro		dec_prepare, rounds, rk, temp
49	load_round_keys	\rk, \rounds, \temp
50	.endm
51
52	.macro		do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
53	aes\de		\i0\().16b, \k\().16b
54	aes\mc		\i0\().16b, \i0\().16b
55	.ifnb		\i1
56	aes\de		\i1\().16b, \k\().16b
57	aes\mc		\i1\().16b, \i1\().16b
58	.ifnb		\i3
59	aes\de		\i2\().16b, \k\().16b
60	aes\mc		\i2\().16b, \i2\().16b
61	aes\de		\i3\().16b, \k\().16b
62	aes\mc		\i3\().16b, \i3\().16b
63	.ifnb		\i4
64	aes\de		\i4\().16b, \k\().16b
65	aes\mc		\i4\().16b, \i4\().16b
66	.endif
67	.endif
68	.endif
69	.endm
70
71	/* up to 5 interleaved encryption rounds with the same round key */
72	.macro		round_Nx, enc, k, i0, i1, i2, i3, i4
73	.ifc		\enc, e
74	do_enc_Nx	e, mc, \k, \i0, \i1, \i2, \i3, \i4
75	.else
76	do_enc_Nx	d, imc, \k, \i0, \i1, \i2, \i3, \i4
77	.endif
78	.endm
79
80	/* up to 5 interleaved final rounds */
81	.macro		fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
82	aes\de		\i0\().16b, \k\().16b
83	.ifnb		\i1
84	aes\de		\i1\().16b, \k\().16b
85	.ifnb		\i3
86	aes\de		\i2\().16b, \k\().16b
87	aes\de		\i3\().16b, \k\().16b
88	.ifnb		\i4
89	aes\de		\i4\().16b, \k\().16b
90	.endif
91	.endif
92	.endif
93	eor		\i0\().16b, \i0\().16b, \k2\().16b
94	.ifnb		\i1
95	eor		\i1\().16b, \i1\().16b, \k2\().16b
96	.ifnb		\i3
97	eor		\i2\().16b, \i2\().16b, \k2\().16b
98	eor		\i3\().16b, \i3\().16b, \k2\().16b
99	.ifnb		\i4
100	eor		\i4\().16b, \i4\().16b, \k2\().16b
101	.endif
102	.endif
103	.endif
104	.endm
105
106	/* up to 5 interleaved blocks */
107	.macro		do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
108	tbz		\rounds, #2, .L\@	/* 128 bits */
109	round_Nx	\enc, v17, \i0, \i1, \i2, \i3, \i4
110	round_Nx	\enc, v18, \i0, \i1, \i2, \i3, \i4
111	tbz		\rounds, #1, .L\@	/* 192 bits */
112	round_Nx	\enc, v19, \i0, \i1, \i2, \i3, \i4
113	round_Nx	\enc, v20, \i0, \i1, \i2, \i3, \i4
114.L\@:	.irp		key, v21, v22, v23, v24, v25, v26, v27, v28, v29
115	round_Nx	\enc, \key, \i0, \i1, \i2, \i3, \i4
116	.endr
117	fin_round_Nx	\enc, v30, v31, \i0, \i1, \i2, \i3, \i4
118	.endm
119
120	.macro		encrypt_block, in, rounds, t0, t1, t2
121	do_block_Nx	e, \rounds, \in
122	.endm
123
124	.macro		encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
125	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3
126	.endm
127
128	.macro		encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
129	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3, \i4
130	.endm
131
132	.macro		decrypt_block, in, rounds, t0, t1, t2
133	do_block_Nx	d, \rounds, \in
134	.endm
135
136	.macro		decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
137	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3
138	.endm
139
140	.macro		decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
141	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3, \i4
142	.endm
143
144#define MAX_STRIDE	5
145
146#include "aes-modes.S"
147