xref: /linux/arch/arm64/crypto/poly1305-armv8.pl (revision 870b7fdc660b38c4e1bd8bf48e62aa352ddf8f42) 
1#!/usr/bin/env perl
2# SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause
3#
4# ====================================================================
5# Written by Andy Polyakov, @dot-asm, initially for the OpenSSL
6# project.
7# ====================================================================
8#
9# This module implements Poly1305 hash for ARMv8.
10#
11# June 2015
12#
13# Numbers are cycles per processed byte with poly1305_blocks alone.
14#
15#		IALU/gcc-4.9	NEON
16#
17# Apple A7	1.86/+5%	0.72
18# Cortex-A53	2.69/+58%	1.47
19# Cortex-A57	2.70/+7%	1.14
20# Denver	1.64/+50%	1.18(*)
21# X-Gene	2.13/+68%	2.27
22# Mongoose	1.77/+75%	1.12
23# Kryo		2.70/+55%	1.13
24# ThunderX2	1.17/+95%	1.36
25#
26# (*)	estimate based on resources availability is less than 1.0,
27#	i.e. measured result is worse than expected, presumably binary
28#	translator is not almighty;
29
30$flavour=shift;
31$output=shift;
32
33if ($flavour && $flavour ne "void") {
34    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
35    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
36    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
37    die "can't locate arm-xlate.pl";
38
39    open STDOUT,"| \"$^X\" $xlate $flavour $output";
40} else {
41    open STDOUT,">$output";
42}
43
44my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
45my ($mac,$nonce)=($inp,$len);
46
47my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));
48
49$code.=<<___;
50#ifndef __KERNEL__
51# include "arm_arch.h"
52.extern	OPENSSL_armcap_P
53#endif
54
55.text
56
57// forward "declarations" are required for Apple
58.globl	poly1305_blocks
59.globl	poly1305_emit
60
61.globl	poly1305_init
62.type	poly1305_init,%function
63.align	5
64poly1305_init:
65	cmp	$inp,xzr
66	stp	xzr,xzr,[$ctx]		// zero hash value
67	stp	xzr,xzr,[$ctx,#16]	// [along with is_base2_26]
68
69	csel	x0,xzr,x0,eq
70	b.eq	.Lno_key
71
72#ifndef	__KERNEL__
73	adrp	x17,OPENSSL_armcap_P
74	ldr	w17,[x17,#:lo12:OPENSSL_armcap_P]
75#endif
76
77	ldp	$r0,$r1,[$inp]		// load key
78	mov	$s1,#0xfffffffc0fffffff
79	movk	$s1,#0x0fff,lsl#48
80#ifdef	__AARCH64EB__
81	rev	$r0,$r0			// flip bytes
82	rev	$r1,$r1
83#endif
84	and	$r0,$r0,$s1		// &=0ffffffc0fffffff
85	and	$s1,$s1,#-4
86	and	$r1,$r1,$s1		// &=0ffffffc0ffffffc
87	mov	w#$s1,#-1
88	stp	$r0,$r1,[$ctx,#32]	// save key value
89	str	w#$s1,[$ctx,#48]	// impossible key power value
90
91#ifndef	__KERNEL__
92	tst	w17,#ARMV7_NEON
93
94	adr	$d0,.Lpoly1305_blocks
95	adr	$r0,.Lpoly1305_blocks_neon
96	adr	$d1,.Lpoly1305_emit
97
98	csel	$d0,$d0,$r0,eq
99
100# ifdef	__ILP32__
101	stp	w#$d0,w#$d1,[$len]
102# else
103	stp	$d0,$d1,[$len]
104# endif
105#endif
106	mov	x0,#1
107.Lno_key:
108	ret
109.size	poly1305_init,.-poly1305_init
110
111.type	poly1305_blocks,%function
112.align	5
113poly1305_blocks:
114.Lpoly1305_blocks:
115	ands	$len,$len,#-16
116	b.eq	.Lno_data
117
118	ldp	$h0,$h1,[$ctx]		// load hash value
119	ldp	$h2,x17,[$ctx,#16]	// [along with is_base2_26]
120	ldp	$r0,$r1,[$ctx,#32]	// load key value
121
122#ifdef	__AARCH64EB__
123	lsr	$d0,$h0,#32
124	mov	w#$d1,w#$h0
125	lsr	$d2,$h1,#32
126	mov	w15,w#$h1
127	lsr	x16,$h2,#32
128#else
129	mov	w#$d0,w#$h0
130	lsr	$d1,$h0,#32
131	mov	w#$d2,w#$h1
132	lsr	x15,$h1,#32
133	mov	w16,w#$h2
134#endif
135
136	add	$d0,$d0,$d1,lsl#26	// base 2^26 -> base 2^64
137	lsr	$d1,$d2,#12
138	adds	$d0,$d0,$d2,lsl#52
139	add	$d1,$d1,x15,lsl#14
140	adc	$d1,$d1,xzr
141	lsr	$d2,x16,#24
142	adds	$d1,$d1,x16,lsl#40
143	adc	$d2,$d2,xzr
144
145	cmp	x17,#0			// is_base2_26?
146	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
147	csel	$h0,$h0,$d0,eq		// choose between radixes
148	csel	$h1,$h1,$d1,eq
149	csel	$h2,$h2,$d2,eq
150
151.Loop:
152	ldp	$t0,$t1,[$inp],#16	// load input
153	sub	$len,$len,#16
154#ifdef	__AARCH64EB__
155	rev	$t0,$t0
156	rev	$t1,$t1
157#endif
158	adds	$h0,$h0,$t0		// accumulate input
159	adcs	$h1,$h1,$t1
160
161	mul	$d0,$h0,$r0		// h0*r0
162	adc	$h2,$h2,$padbit
163	umulh	$d1,$h0,$r0
164
165	mul	$t0,$h1,$s1		// h1*5*r1
166	umulh	$t1,$h1,$s1
167
168	adds	$d0,$d0,$t0
169	mul	$t0,$h0,$r1		// h0*r1
170	adc	$d1,$d1,$t1
171	umulh	$d2,$h0,$r1
172
173	adds	$d1,$d1,$t0
174	mul	$t0,$h1,$r0		// h1*r0
175	adc	$d2,$d2,xzr
176	umulh	$t1,$h1,$r0
177
178	adds	$d1,$d1,$t0
179	mul	$t0,$h2,$s1		// h2*5*r1
180	adc	$d2,$d2,$t1
181	mul	$t1,$h2,$r0		// h2*r0
182
183	adds	$d1,$d1,$t0
184	adc	$d2,$d2,$t1
185
186	and	$t0,$d2,#-4		// final reduction
187	and	$h2,$d2,#3
188	add	$t0,$t0,$d2,lsr#2
189	adds	$h0,$d0,$t0
190	adcs	$h1,$d1,xzr
191	adc	$h2,$h2,xzr
192
193	cbnz	$len,.Loop
194
195	stp	$h0,$h1,[$ctx]		// store hash value
196	stp	$h2,xzr,[$ctx,#16]	// [and clear is_base2_26]
197
198.Lno_data:
199	ret
200.size	poly1305_blocks,.-poly1305_blocks
201
202.type	poly1305_emit,%function
203.align	5
204poly1305_emit:
205.Lpoly1305_emit:
206	ldp	$h0,$h1,[$ctx]		// load hash base 2^64
207	ldp	$h2,$r0,[$ctx,#16]	// [along with is_base2_26]
208	ldp	$t0,$t1,[$nonce]	// load nonce
209
210#ifdef	__AARCH64EB__
211	lsr	$d0,$h0,#32
212	mov	w#$d1,w#$h0
213	lsr	$d2,$h1,#32
214	mov	w15,w#$h1
215	lsr	x16,$h2,#32
216#else
217	mov	w#$d0,w#$h0
218	lsr	$d1,$h0,#32
219	mov	w#$d2,w#$h1
220	lsr	x15,$h1,#32
221	mov	w16,w#$h2
222#endif
223
224	add	$d0,$d0,$d1,lsl#26	// base 2^26 -> base 2^64
225	lsr	$d1,$d2,#12
226	adds	$d0,$d0,$d2,lsl#52
227	add	$d1,$d1,x15,lsl#14
228	adc	$d1,$d1,xzr
229	lsr	$d2,x16,#24
230	adds	$d1,$d1,x16,lsl#40
231	adc	$d2,$d2,xzr
232
233	cmp	$r0,#0			// is_base2_26?
234	csel	$h0,$h0,$d0,eq		// choose between radixes
235	csel	$h1,$h1,$d1,eq
236	csel	$h2,$h2,$d2,eq
237
238	adds	$d0,$h0,#5		// compare to modulus
239	adcs	$d1,$h1,xzr
240	adc	$d2,$h2,xzr
241
242	tst	$d2,#-4			// see if it's carried/borrowed
243
244	csel	$h0,$h0,$d0,eq
245	csel	$h1,$h1,$d1,eq
246
247#ifdef	__AARCH64EB__
248	ror	$t0,$t0,#32		// flip nonce words
249	ror	$t1,$t1,#32
250#endif
251	adds	$h0,$h0,$t0		// accumulate nonce
252	adc	$h1,$h1,$t1
253#ifdef	__AARCH64EB__
254	rev	$h0,$h0			// flip output bytes
255	rev	$h1,$h1
256#endif
257	stp	$h0,$h1,[$mac]		// write result
258
259	ret
260.size	poly1305_emit,.-poly1305_emit
261___
262my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
263my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
264my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
265my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
266my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
267my ($T0,$T1,$MASK) = map("v$_",(29..31));
268
269my ($in2,$zeros)=("x16","x17");
270my $is_base2_26 = $zeros;		# borrow
271
272$code.=<<___;
273.type	poly1305_mult,%function
274.align	5
275poly1305_mult:
276	mul	$d0,$h0,$r0		// h0*r0
277	umulh	$d1,$h0,$r0
278
279	mul	$t0,$h1,$s1		// h1*5*r1
280	umulh	$t1,$h1,$s1
281
282	adds	$d0,$d0,$t0
283	mul	$t0,$h0,$r1		// h0*r1
284	adc	$d1,$d1,$t1
285	umulh	$d2,$h0,$r1
286
287	adds	$d1,$d1,$t0
288	mul	$t0,$h1,$r0		// h1*r0
289	adc	$d2,$d2,xzr
290	umulh	$t1,$h1,$r0
291
292	adds	$d1,$d1,$t0
293	mul	$t0,$h2,$s1		// h2*5*r1
294	adc	$d2,$d2,$t1
295	mul	$t1,$h2,$r0		// h2*r0
296
297	adds	$d1,$d1,$t0
298	adc	$d2,$d2,$t1
299
300	and	$t0,$d2,#-4		// final reduction
301	and	$h2,$d2,#3
302	add	$t0,$t0,$d2,lsr#2
303	adds	$h0,$d0,$t0
304	adcs	$h1,$d1,xzr
305	adc	$h2,$h2,xzr
306
307	ret
308.size	poly1305_mult,.-poly1305_mult
309
310.type	poly1305_splat,%function
311.align	4
312poly1305_splat:
313	and	x12,$h0,#0x03ffffff	// base 2^64 -> base 2^26
314	ubfx	x13,$h0,#26,#26
315	extr	x14,$h1,$h0,#52
316	and	x14,x14,#0x03ffffff
317	ubfx	x15,$h1,#14,#26
318	extr	x16,$h2,$h1,#40
319
320	str	w12,[$ctx,#16*0]	// r0
321	add	w12,w13,w13,lsl#2	// r1*5
322	str	w13,[$ctx,#16*1]	// r1
323	add	w13,w14,w14,lsl#2	// r2*5
324	str	w12,[$ctx,#16*2]	// s1
325	str	w14,[$ctx,#16*3]	// r2
326	add	w14,w15,w15,lsl#2	// r3*5
327	str	w13,[$ctx,#16*4]	// s2
328	str	w15,[$ctx,#16*5]	// r3
329	add	w15,w16,w16,lsl#2	// r4*5
330	str	w14,[$ctx,#16*6]	// s3
331	str	w16,[$ctx,#16*7]	// r4
332	str	w15,[$ctx,#16*8]	// s4
333
334	ret
335.size	poly1305_splat,.-poly1305_splat
336
337#ifdef	__KERNEL__
338.globl	poly1305_blocks_neon
339#endif
340.type	poly1305_blocks_neon,%function
341.align	5
342poly1305_blocks_neon:
343.Lpoly1305_blocks_neon:
344	ldr	$is_base2_26,[$ctx,#24]
345	cmp	$len,#128
346	b.lo	.Lpoly1305_blocks
347
348	.inst	0xd503233f		// paciasp
349	stp	x29,x30,[sp,#-80]!
350	add	x29,sp,#0
351
352	stp	d8,d9,[sp,#16]		// meet ABI requirements
353	stp	d10,d11,[sp,#32]
354	stp	d12,d13,[sp,#48]
355	stp	d14,d15,[sp,#64]
356
357	cbz	$is_base2_26,.Lbase2_64_neon
358
359	ldp	w10,w11,[$ctx]		// load hash value base 2^26
360	ldp	w12,w13,[$ctx,#8]
361	ldr	w14,[$ctx,#16]
362
363	tst	$len,#31
364	b.eq	.Leven_neon
365
366	ldp	$r0,$r1,[$ctx,#32]	// load key value
367
368	add	$h0,x10,x11,lsl#26	// base 2^26 -> base 2^64
369	lsr	$h1,x12,#12
370	adds	$h0,$h0,x12,lsl#52
371	add	$h1,$h1,x13,lsl#14
372	adc	$h1,$h1,xzr
373	lsr	$h2,x14,#24
374	adds	$h1,$h1,x14,lsl#40
375	adc	$d2,$h2,xzr		// can be partially reduced...
376
377	ldp	$d0,$d1,[$inp],#16	// load input
378	sub	$len,$len,#16
379	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
380
381#ifdef	__AARCH64EB__
382	rev	$d0,$d0
383	rev	$d1,$d1
384#endif
385	adds	$h0,$h0,$d0		// accumulate input
386	adcs	$h1,$h1,$d1
387	adc	$h2,$h2,$padbit
388
389	bl	poly1305_mult
390
391	and	x10,$h0,#0x03ffffff	// base 2^64 -> base 2^26
392	ubfx	x11,$h0,#26,#26
393	extr	x12,$h1,$h0,#52
394	and	x12,x12,#0x03ffffff
395	ubfx	x13,$h1,#14,#26
396	extr	x14,$h2,$h1,#40
397
398	b	.Leven_neon
399
400.align	4
401.Lbase2_64_neon:
402	ldp	$r0,$r1,[$ctx,#32]	// load key value
403
404	ldp	$h0,$h1,[$ctx]		// load hash value base 2^64
405	ldr	$h2,[$ctx,#16]
406
407	tst	$len,#31
408	b.eq	.Linit_neon
409
410	ldp	$d0,$d1,[$inp],#16	// load input
411	sub	$len,$len,#16
412	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
413#ifdef	__AARCH64EB__
414	rev	$d0,$d0
415	rev	$d1,$d1
416#endif
417	adds	$h0,$h0,$d0		// accumulate input
418	adcs	$h1,$h1,$d1
419	adc	$h2,$h2,$padbit
420
421	bl	poly1305_mult
422
423.Linit_neon:
424	ldr	w17,[$ctx,#48]		// first table element
425	and	x10,$h0,#0x03ffffff	// base 2^64 -> base 2^26
426	ubfx	x11,$h0,#26,#26
427	extr	x12,$h1,$h0,#52
428	and	x12,x12,#0x03ffffff
429	ubfx	x13,$h1,#14,#26
430	extr	x14,$h2,$h1,#40
431
432	cmp	w17,#-1			// is value impossible?
433	b.ne	.Leven_neon
434
435	fmov	${H0},x10
436	fmov	${H1},x11
437	fmov	${H2},x12
438	fmov	${H3},x13
439	fmov	${H4},x14
440
441	////////////////////////////////// initialize r^n table
442	mov	$h0,$r0			// r^1
443	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
444	mov	$h1,$r1
445	mov	$h2,xzr
446	add	$ctx,$ctx,#48+12
447	bl	poly1305_splat
448
449	bl	poly1305_mult		// r^2
450	sub	$ctx,$ctx,#4
451	bl	poly1305_splat
452
453	bl	poly1305_mult		// r^3
454	sub	$ctx,$ctx,#4
455	bl	poly1305_splat
456
457	bl	poly1305_mult		// r^4
458	sub	$ctx,$ctx,#4
459	bl	poly1305_splat
460	sub	$ctx,$ctx,#48		// restore original $ctx
461	b	.Ldo_neon
462
463.align	4
464.Leven_neon:
465	fmov	${H0},x10
466	fmov	${H1},x11
467	fmov	${H2},x12
468	fmov	${H3},x13
469	fmov	${H4},x14
470
471.Ldo_neon:
472	ldp	x8,x12,[$inp,#32]	// inp[2:3]
473	subs	$len,$len,#64
474	ldp	x9,x13,[$inp,#48]
475	add	$in2,$inp,#96
476	adrp	$zeros,.Lzeros
477	add	$zeros,$zeros,#:lo12:.Lzeros
478
479	lsl	$padbit,$padbit,#24
480	add	x15,$ctx,#48
481
482#ifdef	__AARCH64EB__
483	rev	x8,x8
484	rev	x12,x12
485	rev	x9,x9
486	rev	x13,x13
487#endif
488	and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
489	and	x5,x9,#0x03ffffff
490	ubfx	x6,x8,#26,#26
491	ubfx	x7,x9,#26,#26
492	add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
493	extr	x8,x12,x8,#52
494	extr	x9,x13,x9,#52
495	add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
496	fmov	$IN23_0,x4
497	and	x8,x8,#0x03ffffff
498	and	x9,x9,#0x03ffffff
499	ubfx	x10,x12,#14,#26
500	ubfx	x11,x13,#14,#26
501	add	x12,$padbit,x12,lsr#40
502	add	x13,$padbit,x13,lsr#40
503	add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
504	fmov	$IN23_1,x6
505	add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
506	add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
507	fmov	$IN23_2,x8
508	fmov	$IN23_3,x10
509	fmov	$IN23_4,x12
510
511	ldp	x8,x12,[$inp],#16	// inp[0:1]
512	ldp	x9,x13,[$inp],#48
513
514	ld1	{$R0,$R1,$S1,$R2},[x15],#64
515	ld1	{$S2,$R3,$S3,$R4},[x15],#64
516	ld1	{$S4},[x15]
517
518#ifdef	__AARCH64EB__
519	rev	x8,x8
520	rev	x12,x12
521	rev	x9,x9
522	rev	x13,x13
523#endif
524	and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
525	and	x5,x9,#0x03ffffff
526	ubfx	x6,x8,#26,#26
527	ubfx	x7,x9,#26,#26
528	add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
529	extr	x8,x12,x8,#52
530	extr	x9,x13,x9,#52
531	add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
532	fmov	$IN01_0,x4
533	and	x8,x8,#0x03ffffff
534	and	x9,x9,#0x03ffffff
535	ubfx	x10,x12,#14,#26
536	ubfx	x11,x13,#14,#26
537	add	x12,$padbit,x12,lsr#40
538	add	x13,$padbit,x13,lsr#40
539	add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
540	fmov	$IN01_1,x6
541	add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
542	add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
543	movi	$MASK.2d,#-1
544	fmov	$IN01_2,x8
545	fmov	$IN01_3,x10
546	fmov	$IN01_4,x12
547	ushr	$MASK.2d,$MASK.2d,#38
548
549	b.ls	.Lskip_loop
550
551.align	4
552.Loop_neon:
553	////////////////////////////////////////////////////////////////
554	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
555	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
556	//   \___________________/
557	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
558	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
559	//   \___________________/ \____________________/
560	//
561	// Note that we start with inp[2:3]*r^2. This is because it
562	// doesn't depend on reduction in previous iteration.
563	////////////////////////////////////////////////////////////////
564	// d4 = h0*r4 + h1*r3   + h2*r2   + h3*r1   + h4*r0
565	// d3 = h0*r3 + h1*r2   + h2*r1   + h3*r0   + h4*5*r4
566	// d2 = h0*r2 + h1*r1   + h2*r0   + h3*5*r4 + h4*5*r3
567	// d1 = h0*r1 + h1*r0   + h2*5*r4 + h3*5*r3 + h4*5*r2
568	// d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
569
570	subs	$len,$len,#64
571	umull	$ACC4,$IN23_0,${R4}[2]
572	csel	$in2,$zeros,$in2,lo
573	umull	$ACC3,$IN23_0,${R3}[2]
574	umull	$ACC2,$IN23_0,${R2}[2]
575	 ldp	x8,x12,[$in2],#16	// inp[2:3] (or zero)
576	umull	$ACC1,$IN23_0,${R1}[2]
577	 ldp	x9,x13,[$in2],#48
578	umull	$ACC0,$IN23_0,${R0}[2]
579#ifdef	__AARCH64EB__
580	 rev	x8,x8
581	 rev	x12,x12
582	 rev	x9,x9
583	 rev	x13,x13
584#endif
585
586	umlal	$ACC4,$IN23_1,${R3}[2]
587	 and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
588	umlal	$ACC3,$IN23_1,${R2}[2]
589	 and	x5,x9,#0x03ffffff
590	umlal	$ACC2,$IN23_1,${R1}[2]
591	 ubfx	x6,x8,#26,#26
592	umlal	$ACC1,$IN23_1,${R0}[2]
593	 ubfx	x7,x9,#26,#26
594	umlal	$ACC0,$IN23_1,${S4}[2]
595	 add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
596
597	umlal	$ACC4,$IN23_2,${R2}[2]
598	 extr	x8,x12,x8,#52
599	umlal	$ACC3,$IN23_2,${R1}[2]
600	 extr	x9,x13,x9,#52
601	umlal	$ACC2,$IN23_2,${R0}[2]
602	 add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
603	umlal	$ACC1,$IN23_2,${S4}[2]
604	 fmov	$IN23_0,x4
605	umlal	$ACC0,$IN23_2,${S3}[2]
606	 and	x8,x8,#0x03ffffff
607
608	umlal	$ACC4,$IN23_3,${R1}[2]
609	 and	x9,x9,#0x03ffffff
610	umlal	$ACC3,$IN23_3,${R0}[2]
611	 ubfx	x10,x12,#14,#26
612	umlal	$ACC2,$IN23_3,${S4}[2]
613	 ubfx	x11,x13,#14,#26
614	umlal	$ACC1,$IN23_3,${S3}[2]
615	 add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
616	umlal	$ACC0,$IN23_3,${S2}[2]
617	 fmov	$IN23_1,x6
618
619	add	$IN01_2,$IN01_2,$H2
620	 add	x12,$padbit,x12,lsr#40
621	umlal	$ACC4,$IN23_4,${R0}[2]
622	 add	x13,$padbit,x13,lsr#40
623	umlal	$ACC3,$IN23_4,${S4}[2]
624	 add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
625	umlal	$ACC2,$IN23_4,${S3}[2]
626	 add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
627	umlal	$ACC1,$IN23_4,${S2}[2]
628	 fmov	$IN23_2,x8
629	umlal	$ACC0,$IN23_4,${S1}[2]
630	 fmov	$IN23_3,x10
631
632	////////////////////////////////////////////////////////////////
633	// (hash+inp[0:1])*r^4 and accumulate
634
635	add	$IN01_0,$IN01_0,$H0
636	 fmov	$IN23_4,x12
637	umlal	$ACC3,$IN01_2,${R1}[0]
638	 ldp	x8,x12,[$inp],#16	// inp[0:1]
639	umlal	$ACC0,$IN01_2,${S3}[0]
640	 ldp	x9,x13,[$inp],#48
641	umlal	$ACC4,$IN01_2,${R2}[0]
642	umlal	$ACC1,$IN01_2,${S4}[0]
643	umlal	$ACC2,$IN01_2,${R0}[0]
644#ifdef	__AARCH64EB__
645	 rev	x8,x8
646	 rev	x12,x12
647	 rev	x9,x9
648	 rev	x13,x13
649#endif
650
651	add	$IN01_1,$IN01_1,$H1
652	umlal	$ACC3,$IN01_0,${R3}[0]
653	umlal	$ACC4,$IN01_0,${R4}[0]
654	 and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
655	umlal	$ACC2,$IN01_0,${R2}[0]
656	 and	x5,x9,#0x03ffffff
657	umlal	$ACC0,$IN01_0,${R0}[0]
658	 ubfx	x6,x8,#26,#26
659	umlal	$ACC1,$IN01_0,${R1}[0]
660	 ubfx	x7,x9,#26,#26
661
662	add	$IN01_3,$IN01_3,$H3
663	 add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
664	umlal	$ACC3,$IN01_1,${R2}[0]
665	 extr	x8,x12,x8,#52
666	umlal	$ACC4,$IN01_1,${R3}[0]
667	 extr	x9,x13,x9,#52
668	umlal	$ACC0,$IN01_1,${S4}[0]
669	 add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
670	umlal	$ACC2,$IN01_1,${R1}[0]
671	 fmov	$IN01_0,x4
672	umlal	$ACC1,$IN01_1,${R0}[0]
673	 and	x8,x8,#0x03ffffff
674
675	add	$IN01_4,$IN01_4,$H4
676	 and	x9,x9,#0x03ffffff
677	umlal	$ACC3,$IN01_3,${R0}[0]
678	 ubfx	x10,x12,#14,#26
679	umlal	$ACC0,$IN01_3,${S2}[0]
680	 ubfx	x11,x13,#14,#26
681	umlal	$ACC4,$IN01_3,${R1}[0]
682	 add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
683	umlal	$ACC1,$IN01_3,${S3}[0]
684	 fmov	$IN01_1,x6
685	umlal	$ACC2,$IN01_3,${S4}[0]
686	 add	x12,$padbit,x12,lsr#40
687
688	umlal	$ACC3,$IN01_4,${S4}[0]
689	 add	x13,$padbit,x13,lsr#40
690	umlal	$ACC0,$IN01_4,${S1}[0]
691	 add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
692	umlal	$ACC4,$IN01_4,${R0}[0]
693	 add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
694	umlal	$ACC1,$IN01_4,${S2}[0]
695	 fmov	$IN01_2,x8
696	umlal	$ACC2,$IN01_4,${S3}[0]
697	 fmov	$IN01_3,x10
698	 fmov	$IN01_4,x12
699
700	/////////////////////////////////////////////////////////////////
701	// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
702	// and P. Schwabe
703	//
704	// [see discussion in poly1305-armv4 module]
705
706	ushr	$T0.2d,$ACC3,#26
707	xtn	$H3,$ACC3
708	 ushr	$T1.2d,$ACC0,#26
709	 and	$ACC0,$ACC0,$MASK.2d
710	add	$ACC4,$ACC4,$T0.2d	// h3 -> h4
711	bic	$H3,#0xfc,lsl#24	// &=0x03ffffff
712	 add	$ACC1,$ACC1,$T1.2d	// h0 -> h1
713
714	ushr	$T0.2d,$ACC4,#26
715	xtn	$H4,$ACC4
716	 ushr	$T1.2d,$ACC1,#26
717	 xtn	$H1,$ACC1
718	bic	$H4,#0xfc,lsl#24
719	 add	$ACC2,$ACC2,$T1.2d	// h1 -> h2
720
721	add	$ACC0,$ACC0,$T0.2d
722	shl	$T0.2d,$T0.2d,#2
723	 shrn	$T1.2s,$ACC2,#26
724	 xtn	$H2,$ACC2
725	add	$ACC0,$ACC0,$T0.2d	// h4 -> h0
726	 bic	$H1,#0xfc,lsl#24
727	 add	$H3,$H3,$T1.2s		// h2 -> h3
728	 bic	$H2,#0xfc,lsl#24
729
730	shrn	$T0.2s,$ACC0,#26
731	xtn	$H0,$ACC0
732	 ushr	$T1.2s,$H3,#26
733	 bic	$H3,#0xfc,lsl#24
734	 bic	$H0,#0xfc,lsl#24
735	add	$H1,$H1,$T0.2s		// h0 -> h1
736	 add	$H4,$H4,$T1.2s		// h3 -> h4
737
738	b.hi	.Loop_neon
739
740.Lskip_loop:
741	dup	$IN23_2,${IN23_2}[0]
742	add	$IN01_2,$IN01_2,$H2
743
744	////////////////////////////////////////////////////////////////
745	// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
746
747	adds	$len,$len,#32
748	b.ne	.Long_tail
749
750	dup	$IN23_2,${IN01_2}[0]
751	add	$IN23_0,$IN01_0,$H0
752	add	$IN23_3,$IN01_3,$H3
753	add	$IN23_1,$IN01_1,$H1
754	add	$IN23_4,$IN01_4,$H4
755
756.Long_tail:
757	dup	$IN23_0,${IN23_0}[0]
758	umull2	$ACC0,$IN23_2,${S3}
759	umull2	$ACC3,$IN23_2,${R1}
760	umull2	$ACC4,$IN23_2,${R2}
761	umull2	$ACC2,$IN23_2,${R0}
762	umull2	$ACC1,$IN23_2,${S4}
763
764	dup	$IN23_1,${IN23_1}[0]
765	umlal2	$ACC0,$IN23_0,${R0}
766	umlal2	$ACC2,$IN23_0,${R2}
767	umlal2	$ACC3,$IN23_0,${R3}
768	umlal2	$ACC4,$IN23_0,${R4}
769	umlal2	$ACC1,$IN23_0,${R1}
770
771	dup	$IN23_3,${IN23_3}[0]
772	umlal2	$ACC0,$IN23_1,${S4}
773	umlal2	$ACC3,$IN23_1,${R2}
774	umlal2	$ACC2,$IN23_1,${R1}
775	umlal2	$ACC4,$IN23_1,${R3}
776	umlal2	$ACC1,$IN23_1,${R0}
777
778	dup	$IN23_4,${IN23_4}[0]
779	umlal2	$ACC3,$IN23_3,${R0}
780	umlal2	$ACC4,$IN23_3,${R1}
781	umlal2	$ACC0,$IN23_3,${S2}
782	umlal2	$ACC1,$IN23_3,${S3}
783	umlal2	$ACC2,$IN23_3,${S4}
784
785	umlal2	$ACC3,$IN23_4,${S4}
786	umlal2	$ACC0,$IN23_4,${S1}
787	umlal2	$ACC4,$IN23_4,${R0}
788	umlal2	$ACC1,$IN23_4,${S2}
789	umlal2	$ACC2,$IN23_4,${S3}
790
791	b.eq	.Lshort_tail
792
793	////////////////////////////////////////////////////////////////
794	// (hash+inp[0:1])*r^4:r^3 and accumulate
795
796	add	$IN01_0,$IN01_0,$H0
797	umlal	$ACC3,$IN01_2,${R1}
798	umlal	$ACC0,$IN01_2,${S3}
799	umlal	$ACC4,$IN01_2,${R2}
800	umlal	$ACC1,$IN01_2,${S4}
801	umlal	$ACC2,$IN01_2,${R0}
802
803	add	$IN01_1,$IN01_1,$H1
804	umlal	$ACC3,$IN01_0,${R3}
805	umlal	$ACC0,$IN01_0,${R0}
806	umlal	$ACC4,$IN01_0,${R4}
807	umlal	$ACC1,$IN01_0,${R1}
808	umlal	$ACC2,$IN01_0,${R2}
809
810	add	$IN01_3,$IN01_3,$H3
811	umlal	$ACC3,$IN01_1,${R2}
812	umlal	$ACC0,$IN01_1,${S4}
813	umlal	$ACC4,$IN01_1,${R3}
814	umlal	$ACC1,$IN01_1,${R0}
815	umlal	$ACC2,$IN01_1,${R1}
816
817	add	$IN01_4,$IN01_4,$H4
818	umlal	$ACC3,$IN01_3,${R0}
819	umlal	$ACC0,$IN01_3,${S2}
820	umlal	$ACC4,$IN01_3,${R1}
821	umlal	$ACC1,$IN01_3,${S3}
822	umlal	$ACC2,$IN01_3,${S4}
823
824	umlal	$ACC3,$IN01_4,${S4}
825	umlal	$ACC0,$IN01_4,${S1}
826	umlal	$ACC4,$IN01_4,${R0}
827	umlal	$ACC1,$IN01_4,${S2}
828	umlal	$ACC2,$IN01_4,${S3}
829
830.Lshort_tail:
831	////////////////////////////////////////////////////////////////
832	// horizontal add
833
834	addp	$ACC3,$ACC3,$ACC3
835	 ldp	d8,d9,[sp,#16]		// meet ABI requirements
836	addp	$ACC0,$ACC0,$ACC0
837	 ldp	d10,d11,[sp,#32]
838	addp	$ACC4,$ACC4,$ACC4
839	 ldp	d12,d13,[sp,#48]
840	addp	$ACC1,$ACC1,$ACC1
841	 ldp	d14,d15,[sp,#64]
842	addp	$ACC2,$ACC2,$ACC2
843	 ldr	x30,[sp,#8]
844
845	////////////////////////////////////////////////////////////////
846	// lazy reduction, but without narrowing
847
848	ushr	$T0.2d,$ACC3,#26
849	and	$ACC3,$ACC3,$MASK.2d
850	 ushr	$T1.2d,$ACC0,#26
851	 and	$ACC0,$ACC0,$MASK.2d
852
853	add	$ACC4,$ACC4,$T0.2d	// h3 -> h4
854	 add	$ACC1,$ACC1,$T1.2d	// h0 -> h1
855
856	ushr	$T0.2d,$ACC4,#26
857	and	$ACC4,$ACC4,$MASK.2d
858	 ushr	$T1.2d,$ACC1,#26
859	 and	$ACC1,$ACC1,$MASK.2d
860	 add	$ACC2,$ACC2,$T1.2d	// h1 -> h2
861
862	add	$ACC0,$ACC0,$T0.2d
863	shl	$T0.2d,$T0.2d,#2
864	 ushr	$T1.2d,$ACC2,#26
865	 and	$ACC2,$ACC2,$MASK.2d
866	add	$ACC0,$ACC0,$T0.2d	// h4 -> h0
867	 add	$ACC3,$ACC3,$T1.2d	// h2 -> h3
868
869	ushr	$T0.2d,$ACC0,#26
870	and	$ACC0,$ACC0,$MASK.2d
871	 ushr	$T1.2d,$ACC3,#26
872	 and	$ACC3,$ACC3,$MASK.2d
873	add	$ACC1,$ACC1,$T0.2d	// h0 -> h1
874	 add	$ACC4,$ACC4,$T1.2d	// h3 -> h4
875
876	////////////////////////////////////////////////////////////////
877	// write the result, can be partially reduced
878
879	st4	{$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
880	mov	x4,#1
881	st1	{$ACC4}[0],[$ctx]
882	str	x4,[$ctx,#8]		// set is_base2_26
883
884	ldr	x29,[sp],#80
885	 .inst	0xd50323bf		// autiasp
886	ret
887.size	poly1305_blocks_neon,.-poly1305_blocks_neon
888
889.pushsection .rodata
890.align	5
891.Lzeros:
892.long	0,0,0,0,0,0,0,0
893.asciz	"Poly1305 for ARMv8, CRYPTOGAMS by \@dot-asm"
894.popsection
895
896.align	2
897#if !defined(__KERNEL__) && !defined(_WIN64)
898.comm	OPENSSL_armcap_P,4,4
899.hidden	OPENSSL_armcap_P
900#endif
901___
902
903foreach (split("\n",$code)) {
904	s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/			or
905	s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/	or
906	(m/\bdup\b/ and (s/\.[24]s/.2d/g or 1))			or
907	(m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1))	or
908	(m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1))		or
909	(m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1))		or
910	(m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));
911
912	s/\.[124]([sd])\[/.$1\[/;
913	s/w#x([0-9]+)/w$1/g;
914
915	print $_,"\n";
916}
917close STDOUT;
918