xref: /linux/arch/arm/crypto/poly1305-armv4.pl (revision 22164fbe274c2dd96d2887fe121896d321000a61) 
1#!/usr/bin/env perl
2# SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause
3#
4# ====================================================================
5# Written by Andy Polyakov, @dot-asm, initially for the OpenSSL
6# project.
7# ====================================================================
8#
9#			IALU(*)/gcc-4.4		NEON
10#
11# ARM11xx(ARMv6)	7.78/+100%		-
12# Cortex-A5		6.35/+130%		3.00
13# Cortex-A8		6.25/+115%		2.36
14# Cortex-A9		5.10/+95%		2.55
15# Cortex-A15		3.85/+85%		1.25(**)
16# Snapdragon S4		5.70/+100%		1.48(**)
17#
18# (*)	this is for -march=armv6, i.e. with bunch of ldrb loading data;
19# (**)	these are trade-off results, they can be improved by ~8% but at
20#	the cost of 15/12% regression on Cortex-A5/A7, it's even possible
21#	to improve Cortex-A9 result, but then A5/A7 loose more than 20%;
22
23$flavour = shift;
24if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
25else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
26
27if ($flavour && $flavour ne "void") {
28    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
29    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
30    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
31    die "can't locate arm-xlate.pl";
32
33    open STDOUT,"| \"$^X\" $xlate $flavour $output";
34} else {
35    open STDOUT,">$output";
36}
37
38($ctx,$inp,$len,$padbit)=map("r$_",(0..3));
39
40$code.=<<___;
41#ifndef	__KERNEL__
42# include "arm_arch.h"
43#else
44# define __ARM_ARCH__ __LINUX_ARM_ARCH__
45# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__
46# define poly1305_init   poly1305_init_arm
47# define poly1305_blocks poly1305_blocks_arm
48# define poly1305_emit   poly1305_emit_arm
49.globl	poly1305_blocks_neon
50#endif
51
52#if defined(__thumb2__)
53.syntax	unified
54.thumb
55#else
56.code	32
57#endif
58
59.text
60
61.globl	poly1305_emit
62.globl	poly1305_blocks
63.globl	poly1305_init
64.type	poly1305_init,%function
65.align	5
66poly1305_init:
67.Lpoly1305_init:
68	stmdb	sp!,{r4-r11}
69
70	eor	r3,r3,r3
71	cmp	$inp,#0
72	str	r3,[$ctx,#0]		@ zero hash value
73	str	r3,[$ctx,#4]
74	str	r3,[$ctx,#8]
75	str	r3,[$ctx,#12]
76	str	r3,[$ctx,#16]
77	str	r3,[$ctx,#36]		@ clear is_base2_26
78	add	$ctx,$ctx,#20
79
80#ifdef	__thumb2__
81	it	eq
82#endif
83	moveq	r0,#0
84	beq	.Lno_key
85
86#if	__ARM_MAX_ARCH__>=7
87	mov	r3,#-1
88	str	r3,[$ctx,#28]		@ impossible key power value
89# ifndef __KERNEL__
90	adr	r11,.Lpoly1305_init
91	ldr	r12,.LOPENSSL_armcap
92# endif
93#endif
94	ldrb	r4,[$inp,#0]
95	mov	r10,#0x0fffffff
96	ldrb	r5,[$inp,#1]
97	and	r3,r10,#-4		@ 0x0ffffffc
98	ldrb	r6,[$inp,#2]
99	ldrb	r7,[$inp,#3]
100	orr	r4,r4,r5,lsl#8
101	ldrb	r5,[$inp,#4]
102	orr	r4,r4,r6,lsl#16
103	ldrb	r6,[$inp,#5]
104	orr	r4,r4,r7,lsl#24
105	ldrb	r7,[$inp,#6]
106	and	r4,r4,r10
107
108#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
109# if !defined(_WIN32)
110	ldr	r12,[r11,r12]		@ OPENSSL_armcap_P
111# endif
112# if defined(__APPLE__) || defined(_WIN32)
113	ldr	r12,[r12]
114# endif
115#endif
116	ldrb	r8,[$inp,#7]
117	orr	r5,r5,r6,lsl#8
118	ldrb	r6,[$inp,#8]
119	orr	r5,r5,r7,lsl#16
120	ldrb	r7,[$inp,#9]
121	orr	r5,r5,r8,lsl#24
122	ldrb	r8,[$inp,#10]
123	and	r5,r5,r3
124
125#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
126	tst	r12,#ARMV7_NEON		@ check for NEON
127# ifdef	__thumb2__
128	adr	r9,.Lpoly1305_blocks_neon
129	adr	r11,.Lpoly1305_blocks
130	it	ne
131	movne	r11,r9
132	adr	r12,.Lpoly1305_emit
133	orr	r11,r11,#1		@ thumb-ify addresses
134	orr	r12,r12,#1
135# else
136	add	r12,r11,#(.Lpoly1305_emit-.Lpoly1305_init)
137	ite	eq
138	addeq	r11,r11,#(.Lpoly1305_blocks-.Lpoly1305_init)
139	addne	r11,r11,#(.Lpoly1305_blocks_neon-.Lpoly1305_init)
140# endif
141#endif
142	ldrb	r9,[$inp,#11]
143	orr	r6,r6,r7,lsl#8
144	ldrb	r7,[$inp,#12]
145	orr	r6,r6,r8,lsl#16
146	ldrb	r8,[$inp,#13]
147	orr	r6,r6,r9,lsl#24
148	ldrb	r9,[$inp,#14]
149	and	r6,r6,r3
150
151	ldrb	r10,[$inp,#15]
152	orr	r7,r7,r8,lsl#8
153	str	r4,[$ctx,#0]
154	orr	r7,r7,r9,lsl#16
155	str	r5,[$ctx,#4]
156	orr	r7,r7,r10,lsl#24
157	str	r6,[$ctx,#8]
158	and	r7,r7,r3
159	str	r7,[$ctx,#12]
160#if	__ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
161	stmia	r2,{r11,r12}		@ fill functions table
162	mov	r0,#1
163#else
164	mov	r0,#0
165#endif
166.Lno_key:
167	ldmia	sp!,{r4-r11}
168#if	__ARM_ARCH__>=5
169	ret				@ bx	lr
170#else
171	tst	lr,#1
172	moveq	pc,lr			@ be binary compatible with V4, yet
173	bx	lr			@ interoperable with Thumb ISA:-)
174#endif
175.size	poly1305_init,.-poly1305_init
176___
177{
178my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
179my ($s1,$s2,$s3)=($r1,$r2,$r3);
180
181$code.=<<___;
182.type	poly1305_blocks,%function
183.align	5
184poly1305_blocks:
185.Lpoly1305_blocks:
186	stmdb	sp!,{r3-r11,lr}
187
188	ands	$len,$len,#-16
189	beq	.Lno_data
190
191	add	$len,$len,$inp		@ end pointer
192	sub	sp,sp,#32
193
194#if __ARM_ARCH__<7
195	ldmia	$ctx,{$h0-$r3}		@ load context
196	add	$ctx,$ctx,#20
197	str	$len,[sp,#16]		@ offload stuff
198	str	$ctx,[sp,#12]
199#else
200	ldr	lr,[$ctx,#36]		@ is_base2_26
201	ldmia	$ctx!,{$h0-$h4}		@ load hash value
202	str	$len,[sp,#16]		@ offload stuff
203	str	$ctx,[sp,#12]
204
205	adds	$r0,$h0,$h1,lsl#26	@ base 2^26 -> base 2^32
206	mov	$r1,$h1,lsr#6
207	adcs	$r1,$r1,$h2,lsl#20
208	mov	$r2,$h2,lsr#12
209	adcs	$r2,$r2,$h3,lsl#14
210	mov	$r3,$h3,lsr#18
211	adcs	$r3,$r3,$h4,lsl#8
212	mov	$len,#0
213	teq	lr,#0
214	str	$len,[$ctx,#16]		@ clear is_base2_26
215	adc	$len,$len,$h4,lsr#24
216
217	itttt	ne
218	movne	$h0,$r0			@ choose between radixes
219	movne	$h1,$r1
220	movne	$h2,$r2
221	movne	$h3,$r3
222	ldmia	$ctx,{$r0-$r3}		@ load key
223	it	ne
224	movne	$h4,$len
225#endif
226
227	mov	lr,$inp
228	cmp	$padbit,#0
229	str	$r1,[sp,#20]
230	str	$r2,[sp,#24]
231	str	$r3,[sp,#28]
232	b	.Loop
233
234.align	4
235.Loop:
236#if __ARM_ARCH__<7
237	ldrb	r0,[lr],#16		@ load input
238# ifdef	__thumb2__
239	it	hi
240# endif
241	addhi	$h4,$h4,#1		@ 1<<128
242	ldrb	r1,[lr,#-15]
243	ldrb	r2,[lr,#-14]
244	ldrb	r3,[lr,#-13]
245	orr	r1,r0,r1,lsl#8
246	ldrb	r0,[lr,#-12]
247	orr	r2,r1,r2,lsl#16
248	ldrb	r1,[lr,#-11]
249	orr	r3,r2,r3,lsl#24
250	ldrb	r2,[lr,#-10]
251	adds	$h0,$h0,r3		@ accumulate input
252
253	ldrb	r3,[lr,#-9]
254	orr	r1,r0,r1,lsl#8
255	ldrb	r0,[lr,#-8]
256	orr	r2,r1,r2,lsl#16
257	ldrb	r1,[lr,#-7]
258	orr	r3,r2,r3,lsl#24
259	ldrb	r2,[lr,#-6]
260	adcs	$h1,$h1,r3
261
262	ldrb	r3,[lr,#-5]
263	orr	r1,r0,r1,lsl#8
264	ldrb	r0,[lr,#-4]
265	orr	r2,r1,r2,lsl#16
266	ldrb	r1,[lr,#-3]
267	orr	r3,r2,r3,lsl#24
268	ldrb	r2,[lr,#-2]
269	adcs	$h2,$h2,r3
270
271	ldrb	r3,[lr,#-1]
272	orr	r1,r0,r1,lsl#8
273	str	lr,[sp,#8]		@ offload input pointer
274	orr	r2,r1,r2,lsl#16
275	add	$s1,$r1,$r1,lsr#2
276	orr	r3,r2,r3,lsl#24
277#else
278	ldr	r0,[lr],#16		@ load input
279	it	hi
280	addhi	$h4,$h4,#1		@ padbit
281	ldr	r1,[lr,#-12]
282	ldr	r2,[lr,#-8]
283	ldr	r3,[lr,#-4]
284# ifdef	__ARMEB__
285	rev	r0,r0
286	rev	r1,r1
287	rev	r2,r2
288	rev	r3,r3
289# endif
290	adds	$h0,$h0,r0		@ accumulate input
291	str	lr,[sp,#8]		@ offload input pointer
292	adcs	$h1,$h1,r1
293	add	$s1,$r1,$r1,lsr#2
294	adcs	$h2,$h2,r2
295#endif
296	add	$s2,$r2,$r2,lsr#2
297	adcs	$h3,$h3,r3
298	add	$s3,$r3,$r3,lsr#2
299
300	umull	r2,r3,$h1,$r0
301	 adc	$h4,$h4,#0
302	umull	r0,r1,$h0,$r0
303	umlal	r2,r3,$h4,$s1
304	umlal	r0,r1,$h3,$s1
305	ldr	$r1,[sp,#20]		@ reload $r1
306	umlal	r2,r3,$h2,$s3
307	umlal	r0,r1,$h1,$s3
308	umlal	r2,r3,$h3,$s2
309	umlal	r0,r1,$h2,$s2
310	umlal	r2,r3,$h0,$r1
311	str	r0,[sp,#0]		@ future $h0
312	 mul	r0,$s2,$h4
313	ldr	$r2,[sp,#24]		@ reload $r2
314	adds	r2,r2,r1		@ d1+=d0>>32
315	 eor	r1,r1,r1
316	adc	lr,r3,#0		@ future $h2
317	str	r2,[sp,#4]		@ future $h1
318
319	mul	r2,$s3,$h4
320	eor	r3,r3,r3
321	umlal	r0,r1,$h3,$s3
322	ldr	$r3,[sp,#28]		@ reload $r3
323	umlal	r2,r3,$h3,$r0
324	umlal	r0,r1,$h2,$r0
325	umlal	r2,r3,$h2,$r1
326	umlal	r0,r1,$h1,$r1
327	umlal	r2,r3,$h1,$r2
328	umlal	r0,r1,$h0,$r2
329	umlal	r2,r3,$h0,$r3
330	ldr	$h0,[sp,#0]
331	mul	$h4,$r0,$h4
332	ldr	$h1,[sp,#4]
333
334	adds	$h2,lr,r0		@ d2+=d1>>32
335	ldr	lr,[sp,#8]		@ reload input pointer
336	adc	r1,r1,#0
337	adds	$h3,r2,r1		@ d3+=d2>>32
338	ldr	r0,[sp,#16]		@ reload end pointer
339	adc	r3,r3,#0
340	add	$h4,$h4,r3		@ h4+=d3>>32
341
342	and	r1,$h4,#-4
343	and	$h4,$h4,#3
344	add	r1,r1,r1,lsr#2		@ *=5
345	adds	$h0,$h0,r1
346	adcs	$h1,$h1,#0
347	adcs	$h2,$h2,#0
348	adcs	$h3,$h3,#0
349	adc	$h4,$h4,#0
350
351	cmp	r0,lr			@ done yet?
352	bhi	.Loop
353
354	ldr	$ctx,[sp,#12]
355	add	sp,sp,#32
356	stmdb	$ctx,{$h0-$h4}		@ store the result
357
358.Lno_data:
359#if	__ARM_ARCH__>=5
360	ldmia	sp!,{r3-r11,pc}
361#else
362	ldmia	sp!,{r3-r11,lr}
363	tst	lr,#1
364	moveq	pc,lr			@ be binary compatible with V4, yet
365	bx	lr			@ interoperable with Thumb ISA:-)
366#endif
367.size	poly1305_blocks,.-poly1305_blocks
368___
369}
370{
371my ($ctx,$mac,$nonce)=map("r$_",(0..2));
372my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
373my $g4=$ctx;
374
375$code.=<<___;
376.type	poly1305_emit,%function
377.align	5
378poly1305_emit:
379.Lpoly1305_emit:
380	stmdb	sp!,{r4-r11}
381
382	ldmia	$ctx,{$h0-$h4}
383
384#if __ARM_ARCH__>=7
385	ldr	ip,[$ctx,#36]		@ is_base2_26
386
387	adds	$g0,$h0,$h1,lsl#26	@ base 2^26 -> base 2^32
388	mov	$g1,$h1,lsr#6
389	adcs	$g1,$g1,$h2,lsl#20
390	mov	$g2,$h2,lsr#12
391	adcs	$g2,$g2,$h3,lsl#14
392	mov	$g3,$h3,lsr#18
393	adcs	$g3,$g3,$h4,lsl#8
394	mov	$g4,#0
395	adc	$g4,$g4,$h4,lsr#24
396
397	tst	ip,ip
398	itttt	ne
399	movne	$h0,$g0
400	movne	$h1,$g1
401	movne	$h2,$g2
402	movne	$h3,$g3
403	it	ne
404	movne	$h4,$g4
405#endif
406
407	adds	$g0,$h0,#5		@ compare to modulus
408	adcs	$g1,$h1,#0
409	adcs	$g2,$h2,#0
410	adcs	$g3,$h3,#0
411	adc	$g4,$h4,#0
412	tst	$g4,#4			@ did it carry/borrow?
413
414#ifdef	__thumb2__
415	it	ne
416#endif
417	movne	$h0,$g0
418	ldr	$g0,[$nonce,#0]
419#ifdef	__thumb2__
420	it	ne
421#endif
422	movne	$h1,$g1
423	ldr	$g1,[$nonce,#4]
424#ifdef	__thumb2__
425	it	ne
426#endif
427	movne	$h2,$g2
428	ldr	$g2,[$nonce,#8]
429#ifdef	__thumb2__
430	it	ne
431#endif
432	movne	$h3,$g3
433	ldr	$g3,[$nonce,#12]
434
435	adds	$h0,$h0,$g0
436	adcs	$h1,$h1,$g1
437	adcs	$h2,$h2,$g2
438	adc	$h3,$h3,$g3
439
440#if __ARM_ARCH__>=7
441# ifdef __ARMEB__
442	rev	$h0,$h0
443	rev	$h1,$h1
444	rev	$h2,$h2
445	rev	$h3,$h3
446# endif
447	str	$h0,[$mac,#0]
448	str	$h1,[$mac,#4]
449	str	$h2,[$mac,#8]
450	str	$h3,[$mac,#12]
451#else
452	strb	$h0,[$mac,#0]
453	mov	$h0,$h0,lsr#8
454	strb	$h1,[$mac,#4]
455	mov	$h1,$h1,lsr#8
456	strb	$h2,[$mac,#8]
457	mov	$h2,$h2,lsr#8
458	strb	$h3,[$mac,#12]
459	mov	$h3,$h3,lsr#8
460
461	strb	$h0,[$mac,#1]
462	mov	$h0,$h0,lsr#8
463	strb	$h1,[$mac,#5]
464	mov	$h1,$h1,lsr#8
465	strb	$h2,[$mac,#9]
466	mov	$h2,$h2,lsr#8
467	strb	$h3,[$mac,#13]
468	mov	$h3,$h3,lsr#8
469
470	strb	$h0,[$mac,#2]
471	mov	$h0,$h0,lsr#8
472	strb	$h1,[$mac,#6]
473	mov	$h1,$h1,lsr#8
474	strb	$h2,[$mac,#10]
475	mov	$h2,$h2,lsr#8
476	strb	$h3,[$mac,#14]
477	mov	$h3,$h3,lsr#8
478
479	strb	$h0,[$mac,#3]
480	strb	$h1,[$mac,#7]
481	strb	$h2,[$mac,#11]
482	strb	$h3,[$mac,#15]
483#endif
484	ldmia	sp!,{r4-r11}
485#if	__ARM_ARCH__>=5
486	ret				@ bx	lr
487#else
488	tst	lr,#1
489	moveq	pc,lr			@ be binary compatible with V4, yet
490	bx	lr			@ interoperable with Thumb ISA:-)
491#endif
492.size	poly1305_emit,.-poly1305_emit
493___
494{
495my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
496my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
497my ($T0,$T1,$MASK) = map("q$_",(15,4,0));
498
499my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));
500
501$code.=<<___;
502#if	__ARM_MAX_ARCH__>=7
503.fpu	neon
504
505.type	poly1305_init_neon,%function
506.align	5
507poly1305_init_neon:
508.Lpoly1305_init_neon:
509	ldr	r3,[$ctx,#48]		@ first table element
510	cmp	r3,#-1			@ is value impossible?
511	bne	.Lno_init_neon
512
513	ldr	r4,[$ctx,#20]		@ load key base 2^32
514	ldr	r5,[$ctx,#24]
515	ldr	r6,[$ctx,#28]
516	ldr	r7,[$ctx,#32]
517
518	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
519	mov	r3,r4,lsr#26
520	mov	r4,r5,lsr#20
521	orr	r3,r3,r5,lsl#6
522	mov	r5,r6,lsr#14
523	orr	r4,r4,r6,lsl#12
524	mov	r6,r7,lsr#8
525	orr	r5,r5,r7,lsl#18
526	and	r3,r3,#0x03ffffff
527	and	r4,r4,#0x03ffffff
528	and	r5,r5,#0x03ffffff
529
530	vdup.32	$R0,r2			@ r^1 in both lanes
531	add	r2,r3,r3,lsl#2		@ *5
532	vdup.32	$R1,r3
533	add	r3,r4,r4,lsl#2
534	vdup.32	$S1,r2
535	vdup.32	$R2,r4
536	add	r4,r5,r5,lsl#2
537	vdup.32	$S2,r3
538	vdup.32	$R3,r5
539	add	r5,r6,r6,lsl#2
540	vdup.32	$S3,r4
541	vdup.32	$R4,r6
542	vdup.32	$S4,r5
543
544	mov	$zeros,#2		@ counter
545
546.Lsquare_neon:
547	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
548	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
549	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
550	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
551	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
552	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
553
554	vmull.u32	$D0,$R0,${R0}[1]
555	vmull.u32	$D1,$R1,${R0}[1]
556	vmull.u32	$D2,$R2,${R0}[1]
557	vmull.u32	$D3,$R3,${R0}[1]
558	vmull.u32	$D4,$R4,${R0}[1]
559
560	vmlal.u32	$D0,$R4,${S1}[1]
561	vmlal.u32	$D1,$R0,${R1}[1]
562	vmlal.u32	$D2,$R1,${R1}[1]
563	vmlal.u32	$D3,$R2,${R1}[1]
564	vmlal.u32	$D4,$R3,${R1}[1]
565
566	vmlal.u32	$D0,$R3,${S2}[1]
567	vmlal.u32	$D1,$R4,${S2}[1]
568	vmlal.u32	$D3,$R1,${R2}[1]
569	vmlal.u32	$D2,$R0,${R2}[1]
570	vmlal.u32	$D4,$R2,${R2}[1]
571
572	vmlal.u32	$D0,$R2,${S3}[1]
573	vmlal.u32	$D3,$R0,${R3}[1]
574	vmlal.u32	$D1,$R3,${S3}[1]
575	vmlal.u32	$D2,$R4,${S3}[1]
576	vmlal.u32	$D4,$R1,${R3}[1]
577
578	vmlal.u32	$D3,$R4,${S4}[1]
579	vmlal.u32	$D0,$R1,${S4}[1]
580	vmlal.u32	$D1,$R2,${S4}[1]
581	vmlal.u32	$D2,$R3,${S4}[1]
582	vmlal.u32	$D4,$R0,${R4}[1]
583
584	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
585	@ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
586	@ and P. Schwabe
587	@
588	@ H0>>+H1>>+H2>>+H3>>+H4
589	@ H3>>+H4>>*5+H0>>+H1
590	@
591	@ Trivia.
592	@
593	@ Result of multiplication of n-bit number by m-bit number is
594	@ n+m bits wide. However! Even though 2^n is a n+1-bit number,
595	@ m-bit number multiplied by 2^n is still n+m bits wide.
596	@
597	@ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
598	@ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
599	@ one is n+1 bits wide.
600	@
601	@ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
602	@ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
603	@ can be 27. However! In cases when their width exceeds 26 bits
604	@ they are limited by 2^26+2^6. This in turn means that *sum*
605	@ of the products with these values can still be viewed as sum
606	@ of 52-bit numbers as long as the amount of addends is not a
607	@ power of 2. For example,
608	@
609	@ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
610	@
611	@ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
612	@ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
613	@ 8 * (2^52) or 2^55. However, the value is then multiplied by
614	@ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
615	@ which is less than 32 * (2^52) or 2^57. And when processing
616	@ data we are looking at triple as many addends...
617	@
618	@ In key setup procedure pre-reduced H0 is limited by 5*4+1 and
619	@ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
620	@ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
621	@ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
622	@ instruction accepts 2x32-bit input and writes 2x64-bit result.
623	@ This means that result of reduction have to be compressed upon
624	@ loop wrap-around. This can be done in the process of reduction
625	@ to minimize amount of instructions [as well as amount of
626	@ 128-bit instructions, which benefits low-end processors], but
627	@ one has to watch for H2 (which is narrower than H0) and 5*H4
628	@ not being wider than 58 bits, so that result of right shift
629	@ by 26 bits fits in 32 bits. This is also useful on x86,
630	@ because it allows to use paddd in place for paddq, which
631	@ benefits Atom, where paddq is ridiculously slow.
632
633	vshr.u64	$T0,$D3,#26
634	vmovn.i64	$D3#lo,$D3
635	 vshr.u64	$T1,$D0,#26
636	 vmovn.i64	$D0#lo,$D0
637	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
638	vbic.i32	$D3#lo,#0xfc000000	@ &=0x03ffffff
639	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
640	 vbic.i32	$D0#lo,#0xfc000000
641
642	vshrn.u64	$T0#lo,$D4,#26
643	vmovn.i64	$D4#lo,$D4
644	 vshr.u64	$T1,$D1,#26
645	 vmovn.i64	$D1#lo,$D1
646	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
647	vbic.i32	$D4#lo,#0xfc000000
648	 vbic.i32	$D1#lo,#0xfc000000
649
650	vadd.i32	$D0#lo,$D0#lo,$T0#lo
651	vshl.u32	$T0#lo,$T0#lo,#2
652	 vshrn.u64	$T1#lo,$D2,#26
653	 vmovn.i64	$D2#lo,$D2
654	vadd.i32	$D0#lo,$D0#lo,$T0#lo	@ h4 -> h0
655	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
656	 vbic.i32	$D2#lo,#0xfc000000
657
658	vshr.u32	$T0#lo,$D0#lo,#26
659	vbic.i32	$D0#lo,#0xfc000000
660	 vshr.u32	$T1#lo,$D3#lo,#26
661	 vbic.i32	$D3#lo,#0xfc000000
662	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
663	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4
664
665	subs		$zeros,$zeros,#1
666	beq		.Lsquare_break_neon
667
668	add		$tbl0,$ctx,#(48+0*9*4)
669	add		$tbl1,$ctx,#(48+1*9*4)
670
671	vtrn.32		$R0,$D0#lo		@ r^2:r^1
672	vtrn.32		$R2,$D2#lo
673	vtrn.32		$R3,$D3#lo
674	vtrn.32		$R1,$D1#lo
675	vtrn.32		$R4,$D4#lo
676
677	vshl.u32	$S2,$R2,#2		@ *5
678	vshl.u32	$S3,$R3,#2
679	vshl.u32	$S1,$R1,#2
680	vshl.u32	$S4,$R4,#2
681	vadd.i32	$S2,$S2,$R2
682	vadd.i32	$S1,$S1,$R1
683	vadd.i32	$S3,$S3,$R3
684	vadd.i32	$S4,$S4,$R4
685
686	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
687	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
688	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
689	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
690	vst1.32		{${S4}[0]},[$tbl0,:32]
691	vst1.32		{${S4}[1]},[$tbl1,:32]
692
693	b		.Lsquare_neon
694
695.align	4
696.Lsquare_break_neon:
697	add		$tbl0,$ctx,#(48+2*4*9)
698	add		$tbl1,$ctx,#(48+3*4*9)
699
700	vmov		$R0,$D0#lo		@ r^4:r^3
701	vshl.u32	$S1,$D1#lo,#2		@ *5
702	vmov		$R1,$D1#lo
703	vshl.u32	$S2,$D2#lo,#2
704	vmov		$R2,$D2#lo
705	vshl.u32	$S3,$D3#lo,#2
706	vmov		$R3,$D3#lo
707	vshl.u32	$S4,$D4#lo,#2
708	vmov		$R4,$D4#lo
709	vadd.i32	$S1,$S1,$D1#lo
710	vadd.i32	$S2,$S2,$D2#lo
711	vadd.i32	$S3,$S3,$D3#lo
712	vadd.i32	$S4,$S4,$D4#lo
713
714	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
715	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
716	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
717	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
718	vst1.32		{${S4}[0]},[$tbl0]
719	vst1.32		{${S4}[1]},[$tbl1]
720
721.Lno_init_neon:
722	ret				@ bx	lr
723.size	poly1305_init_neon,.-poly1305_init_neon
724
725.type	poly1305_blocks_neon,%function
726.align	5
727poly1305_blocks_neon:
728.Lpoly1305_blocks_neon:
729	ldr	ip,[$ctx,#36]		@ is_base2_26
730
731	cmp	$len,#64
732	blo	.Lpoly1305_blocks
733
734	stmdb	sp!,{r4-r7}
735	vstmdb	sp!,{d8-d15}		@ ABI specification says so
736
737	tst	ip,ip			@ is_base2_26?
738	bne	.Lbase2_26_neon
739
740	stmdb	sp!,{r1-r3,lr}
741	bl	.Lpoly1305_init_neon
742
743	ldr	r4,[$ctx,#0]		@ load hash value base 2^32
744	ldr	r5,[$ctx,#4]
745	ldr	r6,[$ctx,#8]
746	ldr	r7,[$ctx,#12]
747	ldr	ip,[$ctx,#16]
748
749	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
750	mov	r3,r4,lsr#26
751	 veor	$D0#lo,$D0#lo,$D0#lo
752	mov	r4,r5,lsr#20
753	orr	r3,r3,r5,lsl#6
754	 veor	$D1#lo,$D1#lo,$D1#lo
755	mov	r5,r6,lsr#14
756	orr	r4,r4,r6,lsl#12
757	 veor	$D2#lo,$D2#lo,$D2#lo
758	mov	r6,r7,lsr#8
759	orr	r5,r5,r7,lsl#18
760	 veor	$D3#lo,$D3#lo,$D3#lo
761	and	r3,r3,#0x03ffffff
762	orr	r6,r6,ip,lsl#24
763	 veor	$D4#lo,$D4#lo,$D4#lo
764	and	r4,r4,#0x03ffffff
765	mov	r1,#1
766	and	r5,r5,#0x03ffffff
767	str	r1,[$ctx,#36]		@ set is_base2_26
768
769	vmov.32	$D0#lo[0],r2
770	vmov.32	$D1#lo[0],r3
771	vmov.32	$D2#lo[0],r4
772	vmov.32	$D3#lo[0],r5
773	vmov.32	$D4#lo[0],r6
774	adr	$zeros,.Lzeros
775
776	ldmia	sp!,{r1-r3,lr}
777	b	.Lhash_loaded
778
779.align	4
780.Lbase2_26_neon:
781	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
782	@ load hash value
783
784	veor		$D0#lo,$D0#lo,$D0#lo
785	veor		$D1#lo,$D1#lo,$D1#lo
786	veor		$D2#lo,$D2#lo,$D2#lo
787	veor		$D3#lo,$D3#lo,$D3#lo
788	veor		$D4#lo,$D4#lo,$D4#lo
789	vld4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
790	adr		$zeros,.Lzeros
791	vld1.32		{$D4#lo[0]},[$ctx]
792	sub		$ctx,$ctx,#16		@ rewind
793
794.Lhash_loaded:
795	add		$in2,$inp,#32
796	mov		$padbit,$padbit,lsl#24
797	tst		$len,#31
798	beq		.Leven
799
800	vld4.32		{$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
801	vmov.32		$H4#lo[0],$padbit
802	sub		$len,$len,#16
803	add		$in2,$inp,#32
804
805# ifdef	__ARMEB__
806	vrev32.8	$H0,$H0
807	vrev32.8	$H3,$H3
808	vrev32.8	$H1,$H1
809	vrev32.8	$H2,$H2
810# endif
811	vsri.u32	$H4#lo,$H3#lo,#8	@ base 2^32 -> base 2^26
812	vshl.u32	$H3#lo,$H3#lo,#18
813
814	vsri.u32	$H3#lo,$H2#lo,#14
815	vshl.u32	$H2#lo,$H2#lo,#12
816	vadd.i32	$H4#hi,$H4#lo,$D4#lo	@ add hash value and move to #hi
817
818	vbic.i32	$H3#lo,#0xfc000000
819	vsri.u32	$H2#lo,$H1#lo,#20
820	vshl.u32	$H1#lo,$H1#lo,#6
821
822	vbic.i32	$H2#lo,#0xfc000000
823	vsri.u32	$H1#lo,$H0#lo,#26
824	vadd.i32	$H3#hi,$H3#lo,$D3#lo
825
826	vbic.i32	$H0#lo,#0xfc000000
827	vbic.i32	$H1#lo,#0xfc000000
828	vadd.i32	$H2#hi,$H2#lo,$D2#lo
829
830	vadd.i32	$H0#hi,$H0#lo,$D0#lo
831	vadd.i32	$H1#hi,$H1#lo,$D1#lo
832
833	mov		$tbl1,$zeros
834	add		$tbl0,$ctx,#48
835
836	cmp		$len,$len
837	b		.Long_tail
838
839.align	4
840.Leven:
841	subs		$len,$len,#64
842	it		lo
843	movlo		$in2,$zeros
844
845	vmov.i32	$H4,#1<<24		@ padbit, yes, always
846	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
847	add		$inp,$inp,#64
848	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
849	add		$in2,$in2,#64
850	itt		hi
851	addhi		$tbl1,$ctx,#(48+1*9*4)
852	addhi		$tbl0,$ctx,#(48+3*9*4)
853
854# ifdef	__ARMEB__
855	vrev32.8	$H0,$H0
856	vrev32.8	$H3,$H3
857	vrev32.8	$H1,$H1
858	vrev32.8	$H2,$H2
859# endif
860	vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
861	vshl.u32	$H3,$H3,#18
862
863	vsri.u32	$H3,$H2,#14
864	vshl.u32	$H2,$H2,#12
865
866	vbic.i32	$H3,#0xfc000000
867	vsri.u32	$H2,$H1,#20
868	vshl.u32	$H1,$H1,#6
869
870	vbic.i32	$H2,#0xfc000000
871	vsri.u32	$H1,$H0,#26
872
873	vbic.i32	$H0,#0xfc000000
874	vbic.i32	$H1,#0xfc000000
875
876	bls		.Lskip_loop
877
878	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^2
879	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4
880	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
881	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
882	b		.Loop_neon
883
884.align	5
885.Loop_neon:
886	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
887	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
888	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
889	@   \___________________/
890	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
891	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
892	@   \___________________/ \____________________/
893	@
894	@ Note that we start with inp[2:3]*r^2. This is because it
895	@ doesn't depend on reduction in previous iteration.
896	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
897	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
898	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
899	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
900	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
901	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
902
903	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
904	@ inp[2:3]*r^2
905
906	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ accumulate inp[0:1]
907	vmull.u32	$D2,$H2#hi,${R0}[1]
908	vadd.i32	$H0#lo,$H0#lo,$D0#lo
909	vmull.u32	$D0,$H0#hi,${R0}[1]
910	vadd.i32	$H3#lo,$H3#lo,$D3#lo
911	vmull.u32	$D3,$H3#hi,${R0}[1]
912	vmlal.u32	$D2,$H1#hi,${R1}[1]
913	vadd.i32	$H1#lo,$H1#lo,$D1#lo
914	vmull.u32	$D1,$H1#hi,${R0}[1]
915
916	vadd.i32	$H4#lo,$H4#lo,$D4#lo
917	vmull.u32	$D4,$H4#hi,${R0}[1]
918	subs		$len,$len,#64
919	vmlal.u32	$D0,$H4#hi,${S1}[1]
920	it		lo
921	movlo		$in2,$zeros
922	vmlal.u32	$D3,$H2#hi,${R1}[1]
923	vld1.32		${S4}[1],[$tbl1,:32]
924	vmlal.u32	$D1,$H0#hi,${R1}[1]
925	vmlal.u32	$D4,$H3#hi,${R1}[1]
926
927	vmlal.u32	$D0,$H3#hi,${S2}[1]
928	vmlal.u32	$D3,$H1#hi,${R2}[1]
929	vmlal.u32	$D4,$H2#hi,${R2}[1]
930	vmlal.u32	$D1,$H4#hi,${S2}[1]
931	vmlal.u32	$D2,$H0#hi,${R2}[1]
932
933	vmlal.u32	$D3,$H0#hi,${R3}[1]
934	vmlal.u32	$D0,$H2#hi,${S3}[1]
935	vmlal.u32	$D4,$H1#hi,${R3}[1]
936	vmlal.u32	$D1,$H3#hi,${S3}[1]
937	vmlal.u32	$D2,$H4#hi,${S3}[1]
938
939	vmlal.u32	$D3,$H4#hi,${S4}[1]
940	vmlal.u32	$D0,$H1#hi,${S4}[1]
941	vmlal.u32	$D4,$H0#hi,${R4}[1]
942	vmlal.u32	$D1,$H2#hi,${S4}[1]
943	vmlal.u32	$D2,$H3#hi,${S4}[1]
944
945	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
946	add		$in2,$in2,#64
947
948	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
949	@ (hash+inp[0:1])*r^4 and accumulate
950
951	vmlal.u32	$D3,$H3#lo,${R0}[0]
952	vmlal.u32	$D0,$H0#lo,${R0}[0]
953	vmlal.u32	$D4,$H4#lo,${R0}[0]
954	vmlal.u32	$D1,$H1#lo,${R0}[0]
955	vmlal.u32	$D2,$H2#lo,${R0}[0]
956	vld1.32		${S4}[0],[$tbl0,:32]
957
958	vmlal.u32	$D3,$H2#lo,${R1}[0]
959	vmlal.u32	$D0,$H4#lo,${S1}[0]
960	vmlal.u32	$D4,$H3#lo,${R1}[0]
961	vmlal.u32	$D1,$H0#lo,${R1}[0]
962	vmlal.u32	$D2,$H1#lo,${R1}[0]
963
964	vmlal.u32	$D3,$H1#lo,${R2}[0]
965	vmlal.u32	$D0,$H3#lo,${S2}[0]
966	vmlal.u32	$D4,$H2#lo,${R2}[0]
967	vmlal.u32	$D1,$H4#lo,${S2}[0]
968	vmlal.u32	$D2,$H0#lo,${R2}[0]
969
970	vmlal.u32	$D3,$H0#lo,${R3}[0]
971	vmlal.u32	$D0,$H2#lo,${S3}[0]
972	vmlal.u32	$D4,$H1#lo,${R3}[0]
973	vmlal.u32	$D1,$H3#lo,${S3}[0]
974	vmlal.u32	$D3,$H4#lo,${S4}[0]
975
976	vmlal.u32	$D2,$H4#lo,${S3}[0]
977	vmlal.u32	$D0,$H1#lo,${S4}[0]
978	vmlal.u32	$D4,$H0#lo,${R4}[0]
979	vmov.i32	$H4,#1<<24		@ padbit, yes, always
980	vmlal.u32	$D1,$H2#lo,${S4}[0]
981	vmlal.u32	$D2,$H3#lo,${S4}[0]
982
983	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
984	add		$inp,$inp,#64
985# ifdef	__ARMEB__
986	vrev32.8	$H0,$H0
987	vrev32.8	$H1,$H1
988	vrev32.8	$H2,$H2
989	vrev32.8	$H3,$H3
990# endif
991
992	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
993	@ lazy reduction interleaved with base 2^32 -> base 2^26 of
994	@ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4.
995
996	vshr.u64	$T0,$D3,#26
997	vmovn.i64	$D3#lo,$D3
998	 vshr.u64	$T1,$D0,#26
999	 vmovn.i64	$D0#lo,$D0
1000	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
1001	vbic.i32	$D3#lo,#0xfc000000
1002	  vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
1003	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
1004	  vshl.u32	$H3,$H3,#18
1005	 vbic.i32	$D0#lo,#0xfc000000
1006
1007	vshrn.u64	$T0#lo,$D4,#26
1008	vmovn.i64	$D4#lo,$D4
1009	 vshr.u64	$T1,$D1,#26
1010	 vmovn.i64	$D1#lo,$D1
1011	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
1012	  vsri.u32	$H3,$H2,#14
1013	vbic.i32	$D4#lo,#0xfc000000
1014	  vshl.u32	$H2,$H2,#12
1015	 vbic.i32	$D1#lo,#0xfc000000
1016
1017	vadd.i32	$D0#lo,$D0#lo,$T0#lo
1018	vshl.u32	$T0#lo,$T0#lo,#2
1019	  vbic.i32	$H3,#0xfc000000
1020	 vshrn.u64	$T1#lo,$D2,#26
1021	 vmovn.i64	$D2#lo,$D2
1022	vaddl.u32	$D0,$D0#lo,$T0#lo	@ h4 -> h0 [widen for a sec]
1023	  vsri.u32	$H2,$H1,#20
1024	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
1025	  vshl.u32	$H1,$H1,#6
1026	 vbic.i32	$D2#lo,#0xfc000000
1027	  vbic.i32	$H2,#0xfc000000
1028
1029	vshrn.u64	$T0#lo,$D0,#26		@ re-narrow
1030	vmovn.i64	$D0#lo,$D0
1031	  vsri.u32	$H1,$H0,#26
1032	  vbic.i32	$H0,#0xfc000000
1033	 vshr.u32	$T1#lo,$D3#lo,#26
1034	 vbic.i32	$D3#lo,#0xfc000000
1035	vbic.i32	$D0#lo,#0xfc000000
1036	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
1037	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4
1038	  vbic.i32	$H1,#0xfc000000
1039
1040	bhi		.Loop_neon
1041
1042.Lskip_loop:
1043	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1044	@ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1045
1046	add		$tbl1,$ctx,#(48+0*9*4)
1047	add		$tbl0,$ctx,#(48+1*9*4)
1048	adds		$len,$len,#32
1049	it		ne
1050	movne		$len,#0
1051	bne		.Long_tail
1052
1053	vadd.i32	$H2#hi,$H2#lo,$D2#lo	@ add hash value and move to #hi
1054	vadd.i32	$H0#hi,$H0#lo,$D0#lo
1055	vadd.i32	$H3#hi,$H3#lo,$D3#lo
1056	vadd.i32	$H1#hi,$H1#lo,$D1#lo
1057	vadd.i32	$H4#hi,$H4#lo,$D4#lo
1058
1059.Long_tail:
1060	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^1
1061	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^2
1062
1063	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ can be redundant
1064	vmull.u32	$D2,$H2#hi,$R0
1065	vadd.i32	$H0#lo,$H0#lo,$D0#lo
1066	vmull.u32	$D0,$H0#hi,$R0
1067	vadd.i32	$H3#lo,$H3#lo,$D3#lo
1068	vmull.u32	$D3,$H3#hi,$R0
1069	vadd.i32	$H1#lo,$H1#lo,$D1#lo
1070	vmull.u32	$D1,$H1#hi,$R0
1071	vadd.i32	$H4#lo,$H4#lo,$D4#lo
1072	vmull.u32	$D4,$H4#hi,$R0
1073
1074	vmlal.u32	$D0,$H4#hi,$S1
1075	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1076	vmlal.u32	$D3,$H2#hi,$R1
1077	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1078	vmlal.u32	$D1,$H0#hi,$R1
1079	vmlal.u32	$D4,$H3#hi,$R1
1080	vmlal.u32	$D2,$H1#hi,$R1
1081
1082	vmlal.u32	$D3,$H1#hi,$R2
1083	vld1.32		${S4}[1],[$tbl1,:32]
1084	vmlal.u32	$D0,$H3#hi,$S2
1085	vld1.32		${S4}[0],[$tbl0,:32]
1086	vmlal.u32	$D4,$H2#hi,$R2
1087	vmlal.u32	$D1,$H4#hi,$S2
1088	vmlal.u32	$D2,$H0#hi,$R2
1089
1090	vmlal.u32	$D3,$H0#hi,$R3
1091	 it		ne
1092	 addne		$tbl1,$ctx,#(48+2*9*4)
1093	vmlal.u32	$D0,$H2#hi,$S3
1094	 it		ne
1095	 addne		$tbl0,$ctx,#(48+3*9*4)
1096	vmlal.u32	$D4,$H1#hi,$R3
1097	vmlal.u32	$D1,$H3#hi,$S3
1098	vmlal.u32	$D2,$H4#hi,$S3
1099
1100	vmlal.u32	$D3,$H4#hi,$S4
1101	 vorn		$MASK,$MASK,$MASK	@ all-ones, can be redundant
1102	vmlal.u32	$D0,$H1#hi,$S4
1103	 vshr.u64	$MASK,$MASK,#38
1104	vmlal.u32	$D4,$H0#hi,$R4
1105	vmlal.u32	$D1,$H2#hi,$S4
1106	vmlal.u32	$D2,$H3#hi,$S4
1107
1108	beq		.Lshort_tail
1109
1110	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1111	@ (hash+inp[0:1])*r^4:r^3 and accumulate
1112
1113	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^3
1114	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4
1115
1116	vmlal.u32	$D2,$H2#lo,$R0
1117	vmlal.u32	$D0,$H0#lo,$R0
1118	vmlal.u32	$D3,$H3#lo,$R0
1119	vmlal.u32	$D1,$H1#lo,$R0
1120	vmlal.u32	$D4,$H4#lo,$R0
1121
1122	vmlal.u32	$D0,$H4#lo,$S1
1123	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1124	vmlal.u32	$D3,$H2#lo,$R1
1125	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1126	vmlal.u32	$D1,$H0#lo,$R1
1127	vmlal.u32	$D4,$H3#lo,$R1
1128	vmlal.u32	$D2,$H1#lo,$R1
1129
1130	vmlal.u32	$D3,$H1#lo,$R2
1131	vld1.32		${S4}[1],[$tbl1,:32]
1132	vmlal.u32	$D0,$H3#lo,$S2
1133	vld1.32		${S4}[0],[$tbl0,:32]
1134	vmlal.u32	$D4,$H2#lo,$R2
1135	vmlal.u32	$D1,$H4#lo,$S2
1136	vmlal.u32	$D2,$H0#lo,$R2
1137
1138	vmlal.u32	$D3,$H0#lo,$R3
1139	vmlal.u32	$D0,$H2#lo,$S3
1140	vmlal.u32	$D4,$H1#lo,$R3
1141	vmlal.u32	$D1,$H3#lo,$S3
1142	vmlal.u32	$D2,$H4#lo,$S3
1143
1144	vmlal.u32	$D3,$H4#lo,$S4
1145	 vorn		$MASK,$MASK,$MASK	@ all-ones
1146	vmlal.u32	$D0,$H1#lo,$S4
1147	 vshr.u64	$MASK,$MASK,#38
1148	vmlal.u32	$D4,$H0#lo,$R4
1149	vmlal.u32	$D1,$H2#lo,$S4
1150	vmlal.u32	$D2,$H3#lo,$S4
1151
1152.Lshort_tail:
1153	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1154	@ horizontal addition
1155
1156	vadd.i64	$D3#lo,$D3#lo,$D3#hi
1157	vadd.i64	$D0#lo,$D0#lo,$D0#hi
1158	vadd.i64	$D4#lo,$D4#lo,$D4#hi
1159	vadd.i64	$D1#lo,$D1#lo,$D1#hi
1160	vadd.i64	$D2#lo,$D2#lo,$D2#hi
1161
1162	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1163	@ lazy reduction, but without narrowing
1164
1165	vshr.u64	$T0,$D3,#26
1166	vand.i64	$D3,$D3,$MASK
1167	 vshr.u64	$T1,$D0,#26
1168	 vand.i64	$D0,$D0,$MASK
1169	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
1170	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
1171
1172	vshr.u64	$T0,$D4,#26
1173	vand.i64	$D4,$D4,$MASK
1174	 vshr.u64	$T1,$D1,#26
1175	 vand.i64	$D1,$D1,$MASK
1176	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
1177
1178	vadd.i64	$D0,$D0,$T0
1179	vshl.u64	$T0,$T0,#2
1180	 vshr.u64	$T1,$D2,#26
1181	 vand.i64	$D2,$D2,$MASK
1182	vadd.i64	$D0,$D0,$T0		@ h4 -> h0
1183	 vadd.i64	$D3,$D3,$T1		@ h2 -> h3
1184
1185	vshr.u64	$T0,$D0,#26
1186	vand.i64	$D0,$D0,$MASK
1187	 vshr.u64	$T1,$D3,#26
1188	 vand.i64	$D3,$D3,$MASK
1189	vadd.i64	$D1,$D1,$T0		@ h0 -> h1
1190	 vadd.i64	$D4,$D4,$T1		@ h3 -> h4
1191
1192	cmp		$len,#0
1193	bne		.Leven
1194
1195	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1196	@ store hash value
1197
1198	vst4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
1199	vst1.32		{$D4#lo[0]},[$ctx]
1200
1201	vldmia	sp!,{d8-d15}			@ epilogue
1202	ldmia	sp!,{r4-r7}
1203	ret					@ bx	lr
1204.size	poly1305_blocks_neon,.-poly1305_blocks_neon
1205
1206.align	5
1207.Lzeros:
1208.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
1209#ifndef	__KERNEL__
1210.LOPENSSL_armcap:
1211# ifdef	_WIN32
1212.word	OPENSSL_armcap_P
1213# else
1214.word	OPENSSL_armcap_P-.Lpoly1305_init
1215# endif
1216.comm	OPENSSL_armcap_P,4,4
1217.hidden	OPENSSL_armcap_P
1218#endif
1219#endif
1220___
1221}	}
1222$code.=<<___;
1223.asciz	"Poly1305 for ARMv4/NEON, CRYPTOGAMS by \@dot-asm"
1224.align	2
1225___
1226
1227foreach (split("\n",$code)) {
1228	s/\`([^\`]*)\`/eval $1/geo;
1229
1230	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
1231	s/\bret\b/bx	lr/go						or
1232	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;	# make it possible to compile with -march=armv4
1233
1234	print $_,"\n";
1235}
1236close STDOUT; # enforce flush
1237