xref: /linux/arch/mips/crypto/chacha-core.S (revision c95baf12f5077419db01313ab61c2aac007d40cd)
1/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2/*
3 * Copyright (C) 2016-2018 René van Dorst <opensource@vdorst.com>. All Rights Reserved.
4 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
5 */
6
7#define MASK_U32		0x3c
8#define CHACHA20_BLOCK_SIZE	64
9#define STACK_SIZE		32
10
11#define X0	$t0
12#define X1	$t1
13#define X2	$t2
14#define X3	$t3
15#define X4	$t4
16#define X5	$t5
17#define X6	$t6
18#define X7	$t7
19#define X8	$t8
20#define X9	$t9
21#define X10	$v1
22#define X11	$s6
23#define X12	$s5
24#define X13	$s4
25#define X14	$s3
26#define X15	$s2
27/* Use regs which are overwritten on exit for Tx so we don't leak clear data. */
28#define T0	$s1
29#define T1	$s0
30#define T(n)	T ## n
31#define X(n)	X ## n
32
33/* Input arguments */
34#define STATE		$a0
35#define OUT		$a1
36#define IN		$a2
37#define BYTES		$a3
38
39/* Output argument */
40/* NONCE[0] is kept in a register and not in memory.
41 * We don't want to touch original value in memory.
42 * Must be incremented every loop iteration.
43 */
44#define NONCE_0		$v0
45
46/* SAVED_X and SAVED_CA are set in the jump table.
47 * Use regs which are overwritten on exit else we don't leak clear data.
48 * They are used to handling the last bytes which are not multiple of 4.
49 */
50#define SAVED_X		X15
51#define SAVED_CA	$s7
52
53#define IS_UNALIGNED	$s7
54
55#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
56#define MSB 0
57#define LSB 3
58#define ROTx rotl
59#define ROTR(n) rotr n, 24
60#define	CPU_TO_LE32(n) \
61	wsbh	n; \
62	rotr	n, 16;
63#else
64#define MSB 3
65#define LSB 0
66#define ROTx rotr
67#define CPU_TO_LE32(n)
68#define ROTR(n)
69#endif
70
71#define FOR_EACH_WORD(x) \
72	x( 0); \
73	x( 1); \
74	x( 2); \
75	x( 3); \
76	x( 4); \
77	x( 5); \
78	x( 6); \
79	x( 7); \
80	x( 8); \
81	x( 9); \
82	x(10); \
83	x(11); \
84	x(12); \
85	x(13); \
86	x(14); \
87	x(15);
88
89#define FOR_EACH_WORD_REV(x) \
90	x(15); \
91	x(14); \
92	x(13); \
93	x(12); \
94	x(11); \
95	x(10); \
96	x( 9); \
97	x( 8); \
98	x( 7); \
99	x( 6); \
100	x( 5); \
101	x( 4); \
102	x( 3); \
103	x( 2); \
104	x( 1); \
105	x( 0);
106
107#define PLUS_ONE_0	 1
108#define PLUS_ONE_1	 2
109#define PLUS_ONE_2	 3
110#define PLUS_ONE_3	 4
111#define PLUS_ONE_4	 5
112#define PLUS_ONE_5	 6
113#define PLUS_ONE_6	 7
114#define PLUS_ONE_7	 8
115#define PLUS_ONE_8	 9
116#define PLUS_ONE_9	10
117#define PLUS_ONE_10	11
118#define PLUS_ONE_11	12
119#define PLUS_ONE_12	13
120#define PLUS_ONE_13	14
121#define PLUS_ONE_14	15
122#define PLUS_ONE_15	16
123#define PLUS_ONE(x)	PLUS_ONE_ ## x
124#define _CONCAT3(a,b,c)	a ## b ## c
125#define CONCAT3(a,b,c)	_CONCAT3(a,b,c)
126
127#define STORE_UNALIGNED(x) \
128CONCAT3(.Lchacha_mips_xor_unaligned_, PLUS_ONE(x), _b: ;) \
129	.if (x != 12); \
130		lw	T0, (x*4)(STATE); \
131	.endif; \
132	lwl	T1, (x*4)+MSB ## (IN); \
133	lwr	T1, (x*4)+LSB ## (IN); \
134	.if (x == 12); \
135		addu	X ## x, NONCE_0; \
136	.else; \
137		addu	X ## x, T0; \
138	.endif; \
139	CPU_TO_LE32(X ## x); \
140	xor	X ## x, T1; \
141	swl	X ## x, (x*4)+MSB ## (OUT); \
142	swr	X ## x, (x*4)+LSB ## (OUT);
143
144#define STORE_ALIGNED(x) \
145CONCAT3(.Lchacha_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \
146	.if (x != 12); \
147		lw	T0, (x*4)(STATE); \
148	.endif; \
149	lw	T1, (x*4) ## (IN); \
150	.if (x == 12); \
151		addu	X ## x, NONCE_0; \
152	.else; \
153		addu	X ## x, T0; \
154	.endif; \
155	CPU_TO_LE32(X ## x); \
156	xor	X ## x, T1; \
157	sw	X ## x, (x*4) ## (OUT);
158
159/* Jump table macro.
160 * Used for setup and handling the last bytes, which are not multiple of 4.
161 * X15 is free to store Xn
162 * Every jumptable entry must be equal in size.
163 */
164#define JMPTBL_ALIGNED(x) \
165.Lchacha_mips_jmptbl_aligned_ ## x: ; \
166	.set	noreorder; \
167	b	.Lchacha_mips_xor_aligned_ ## x ## _b; \
168	.if (x == 12); \
169		addu	SAVED_X, X ## x, NONCE_0; \
170	.else; \
171		addu	SAVED_X, X ## x, SAVED_CA; \
172	.endif; \
173	.set	reorder
174
175#define JMPTBL_UNALIGNED(x) \
176.Lchacha_mips_jmptbl_unaligned_ ## x: ; \
177	.set	noreorder; \
178	b	.Lchacha_mips_xor_unaligned_ ## x ## _b; \
179	.if (x == 12); \
180		addu	SAVED_X, X ## x, NONCE_0; \
181	.else; \
182		addu	SAVED_X, X ## x, SAVED_CA; \
183	.endif; \
184	.set	reorder
185
186#define AXR(A, B, C, D,  K, L, M, N,  V, W, Y, Z,  S) \
187	addu	X(A), X(K); \
188	addu	X(B), X(L); \
189	addu	X(C), X(M); \
190	addu	X(D), X(N); \
191	xor	X(V), X(A); \
192	xor	X(W), X(B); \
193	xor	X(Y), X(C); \
194	xor	X(Z), X(D); \
195	rotl	X(V), S;    \
196	rotl	X(W), S;    \
197	rotl	X(Y), S;    \
198	rotl	X(Z), S;
199
200.text
201.set	reorder
202.set	noat
203.globl	chacha_crypt_arch
204.ent	chacha_crypt_arch
205chacha_crypt_arch:
206	.frame	$sp, STACK_SIZE, $ra
207
208	/* Load number of rounds */
209	lw	$at, 16($sp)
210
211	addiu	$sp, -STACK_SIZE
212
213	/* Return bytes = 0. */
214	beqz	BYTES, .Lchacha_mips_end
215
216	lw	NONCE_0, 48(STATE)
217
218	/* Save s0-s7 */
219	sw	$s0,  0($sp)
220	sw	$s1,  4($sp)
221	sw	$s2,  8($sp)
222	sw	$s3, 12($sp)
223	sw	$s4, 16($sp)
224	sw	$s5, 20($sp)
225	sw	$s6, 24($sp)
226	sw	$s7, 28($sp)
227
228	/* Test IN or OUT is unaligned.
229	 * IS_UNALIGNED = ( IN | OUT ) & 0x00000003
230	 */
231	or	IS_UNALIGNED, IN, OUT
232	andi	IS_UNALIGNED, 0x3
233
234	b	.Lchacha_rounds_start
235
236.align 4
237.Loop_chacha_rounds:
238	addiu	IN,  CHACHA20_BLOCK_SIZE
239	addiu	OUT, CHACHA20_BLOCK_SIZE
240	addiu	NONCE_0, 1
241
242.Lchacha_rounds_start:
243	lw	X0,  0(STATE)
244	lw	X1,  4(STATE)
245	lw	X2,  8(STATE)
246	lw	X3,  12(STATE)
247
248	lw	X4,  16(STATE)
249	lw	X5,  20(STATE)
250	lw	X6,  24(STATE)
251	lw	X7,  28(STATE)
252	lw	X8,  32(STATE)
253	lw	X9,  36(STATE)
254	lw	X10, 40(STATE)
255	lw	X11, 44(STATE)
256
257	move	X12, NONCE_0
258	lw	X13, 52(STATE)
259	lw	X14, 56(STATE)
260	lw	X15, 60(STATE)
261
262.Loop_chacha_xor_rounds:
263	addiu	$at, -2
264	AXR( 0, 1, 2, 3,  4, 5, 6, 7, 12,13,14,15, 16);
265	AXR( 8, 9,10,11, 12,13,14,15,  4, 5, 6, 7, 12);
266	AXR( 0, 1, 2, 3,  4, 5, 6, 7, 12,13,14,15,  8);
267	AXR( 8, 9,10,11, 12,13,14,15,  4, 5, 6, 7,  7);
268	AXR( 0, 1, 2, 3,  5, 6, 7, 4, 15,12,13,14, 16);
269	AXR(10,11, 8, 9, 15,12,13,14,  5, 6, 7, 4, 12);
270	AXR( 0, 1, 2, 3,  5, 6, 7, 4, 15,12,13,14,  8);
271	AXR(10,11, 8, 9, 15,12,13,14,  5, 6, 7, 4,  7);
272	bnez	$at, .Loop_chacha_xor_rounds
273
274	addiu	BYTES, -(CHACHA20_BLOCK_SIZE)
275
276	/* Is data src/dst unaligned? Jump */
277	bnez	IS_UNALIGNED, .Loop_chacha_unaligned
278
279	/* Set number rounds here to fill delayslot. */
280	lw	$at, (STACK_SIZE+16)($sp)
281
282	/* BYTES < 0, it has no full block. */
283	bltz	BYTES, .Lchacha_mips_no_full_block_aligned
284
285	FOR_EACH_WORD_REV(STORE_ALIGNED)
286
287	/* BYTES > 0? Loop again. */
288	bgtz	BYTES, .Loop_chacha_rounds
289
290	/* Place this here to fill delay slot */
291	addiu	NONCE_0, 1
292
293	/* BYTES < 0? Handle last bytes */
294	bltz	BYTES, .Lchacha_mips_xor_bytes
295
296.Lchacha_mips_xor_done:
297	/* Restore used registers */
298	lw	$s0,  0($sp)
299	lw	$s1,  4($sp)
300	lw	$s2,  8($sp)
301	lw	$s3, 12($sp)
302	lw	$s4, 16($sp)
303	lw	$s5, 20($sp)
304	lw	$s6, 24($sp)
305	lw	$s7, 28($sp)
306
307	/* Write NONCE_0 back to right location in state */
308	sw	NONCE_0, 48(STATE)
309
310.Lchacha_mips_end:
311	addiu	$sp, STACK_SIZE
312	jr	$ra
313
314.Lchacha_mips_no_full_block_aligned:
315	/* Restore the offset on BYTES */
316	addiu	BYTES, CHACHA20_BLOCK_SIZE
317
318	/* Get number of full WORDS */
319	andi	$at, BYTES, MASK_U32
320
321	/* Load upper half of jump table addr */
322	lui	T0, %hi(.Lchacha_mips_jmptbl_aligned_0)
323
324	/* Calculate lower half jump table offset */
325	ins	T0, $at, 1, 6
326
327	/* Add offset to STATE */
328	addu	T1, STATE, $at
329
330	/* Add lower half jump table addr */
331	addiu	T0, %lo(.Lchacha_mips_jmptbl_aligned_0)
332
333	/* Read value from STATE */
334	lw	SAVED_CA, 0(T1)
335
336	/* Store remaining bytecounter as negative value */
337	subu	BYTES, $at, BYTES
338
339	jr	T0
340
341	/* Jump table */
342	FOR_EACH_WORD(JMPTBL_ALIGNED)
343
344
345.Loop_chacha_unaligned:
346	/* Set number rounds here to fill delayslot. */
347	lw	$at, (STACK_SIZE+16)($sp)
348
349	/* BYTES > 0, it has no full block. */
350	bltz	BYTES, .Lchacha_mips_no_full_block_unaligned
351
352	FOR_EACH_WORD_REV(STORE_UNALIGNED)
353
354	/* BYTES > 0? Loop again. */
355	bgtz	BYTES, .Loop_chacha_rounds
356
357	/* Write NONCE_0 back to right location in state */
358	sw	NONCE_0, 48(STATE)
359
360	.set noreorder
361	/* Fall through to byte handling */
362	bgez	BYTES, .Lchacha_mips_xor_done
363.Lchacha_mips_xor_unaligned_0_b:
364.Lchacha_mips_xor_aligned_0_b:
365	/* Place this here to fill delay slot */
366	addiu	NONCE_0, 1
367	.set reorder
368
369.Lchacha_mips_xor_bytes:
370	addu	IN, $at
371	addu	OUT, $at
372	/* First byte */
373	lbu	T1, 0(IN)
374	addiu	$at, BYTES, 1
375	CPU_TO_LE32(SAVED_X)
376	ROTR(SAVED_X)
377	xor	T1, SAVED_X
378	sb	T1, 0(OUT)
379	beqz	$at, .Lchacha_mips_xor_done
380	/* Second byte */
381	lbu	T1, 1(IN)
382	addiu	$at, BYTES, 2
383	ROTx	SAVED_X, 8
384	xor	T1, SAVED_X
385	sb	T1, 1(OUT)
386	beqz	$at, .Lchacha_mips_xor_done
387	/* Third byte */
388	lbu	T1, 2(IN)
389	ROTx	SAVED_X, 8
390	xor	T1, SAVED_X
391	sb	T1, 2(OUT)
392	b	.Lchacha_mips_xor_done
393
394.Lchacha_mips_no_full_block_unaligned:
395	/* Restore the offset on BYTES */
396	addiu	BYTES, CHACHA20_BLOCK_SIZE
397
398	/* Get number of full WORDS */
399	andi	$at, BYTES, MASK_U32
400
401	/* Load upper half of jump table addr */
402	lui	T0, %hi(.Lchacha_mips_jmptbl_unaligned_0)
403
404	/* Calculate lower half jump table offset */
405	ins	T0, $at, 1, 6
406
407	/* Add offset to STATE */
408	addu	T1, STATE, $at
409
410	/* Add lower half jump table addr */
411	addiu	T0, %lo(.Lchacha_mips_jmptbl_unaligned_0)
412
413	/* Read value from STATE */
414	lw	SAVED_CA, 0(T1)
415
416	/* Store remaining bytecounter as negative value */
417	subu	BYTES, $at, BYTES
418
419	jr	T0
420
421	/* Jump table */
422	FOR_EACH_WORD(JMPTBL_UNALIGNED)
423.end chacha_crypt_arch
424.set at
425
426/* Input arguments
427 * STATE	$a0
428 * OUT		$a1
429 * NROUND	$a2
430 */
431
432#undef X12
433#undef X13
434#undef X14
435#undef X15
436
437#define X12	$a3
438#define X13	$at
439#define X14	$v0
440#define X15	STATE
441
442.set noat
443.globl	hchacha_block_arch
444.ent	hchacha_block_arch
445hchacha_block_arch:
446	.frame	$sp, STACK_SIZE, $ra
447
448	addiu	$sp, -STACK_SIZE
449
450	/* Save X11(s6) */
451	sw	X11, 0($sp)
452
453	lw	X0,  0(STATE)
454	lw	X1,  4(STATE)
455	lw	X2,  8(STATE)
456	lw	X3,  12(STATE)
457	lw	X4,  16(STATE)
458	lw	X5,  20(STATE)
459	lw	X6,  24(STATE)
460	lw	X7,  28(STATE)
461	lw	X8,  32(STATE)
462	lw	X9,  36(STATE)
463	lw	X10, 40(STATE)
464	lw	X11, 44(STATE)
465	lw	X12, 48(STATE)
466	lw	X13, 52(STATE)
467	lw	X14, 56(STATE)
468	lw	X15, 60(STATE)
469
470.Loop_hchacha_xor_rounds:
471	addiu	$a2, -2
472	AXR( 0, 1, 2, 3,  4, 5, 6, 7, 12,13,14,15, 16);
473	AXR( 8, 9,10,11, 12,13,14,15,  4, 5, 6, 7, 12);
474	AXR( 0, 1, 2, 3,  4, 5, 6, 7, 12,13,14,15,  8);
475	AXR( 8, 9,10,11, 12,13,14,15,  4, 5, 6, 7,  7);
476	AXR( 0, 1, 2, 3,  5, 6, 7, 4, 15,12,13,14, 16);
477	AXR(10,11, 8, 9, 15,12,13,14,  5, 6, 7, 4, 12);
478	AXR( 0, 1, 2, 3,  5, 6, 7, 4, 15,12,13,14,  8);
479	AXR(10,11, 8, 9, 15,12,13,14,  5, 6, 7, 4,  7);
480	bnez	$a2, .Loop_hchacha_xor_rounds
481
482	/* Restore used register */
483	lw	X11, 0($sp)
484
485	sw	X0,  0(OUT)
486	sw	X1,  4(OUT)
487	sw	X2,  8(OUT)
488	sw	X3,  12(OUT)
489	sw	X12, 16(OUT)
490	sw	X13, 20(OUT)
491	sw	X14, 24(OUT)
492	sw	X15, 28(OUT)
493
494	addiu	$sp, STACK_SIZE
495	jr	$ra
496.end hchacha_block_arch
497.set at
498