VirtualBox

source: vbox/trunk/src/libs/openssl-1.1.1f/crypto/poly1305/asm/poly1305-armv8.pl@ 83531

Last change on this file since 83531 was 83531, checked in by vboxsync, 5 years ago

setting svn:sync-process=export for openssl-1.1.1f, all files except tests
  • Property svn:executable set to *
File size: 21.1 KB
Line 
1#! /usr/bin/env perl
2# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
3#
4# Licensed under the OpenSSL license (the "License"). You may not use
5# this file except in compliance with the License. You can obtain a copy
6# in the file LICENSE in the source distribution or at
7# https://www.openssl.org/source/license.html
8
9#
10# ====================================================================
11# Written by Andy Polyakov <[email protected]> for the OpenSSL
12# project. The module is, however, dual licensed under OpenSSL and
13# CRYPTOGAMS licenses depending on where you obtain it. For further
14# details see http://www.openssl.org/~appro/cryptogams/.
15# ====================================================================
16#
17# This module implements Poly1305 hash for ARMv8.
18#
19# June 2015
20#
21# Numbers are cycles per processed byte with poly1305_blocks alone.
22#
23# IALU/gcc-4.9 NEON
24#
25# Apple A7 1.86/+5% 0.72
26# Cortex-A53 2.69/+58% 1.47
27# Cortex-A57 2.70/+7% 1.14
28# Denver 1.64/+50% 1.18(*)
29# X-Gene 2.13/+68% 2.27
30# Mongoose 1.77/+75% 1.12
31# Kryo 2.70/+55% 1.13
32#
33# (*) estimate based on resources availability is less than 1.0,
34# i.e. measured result is worse than expected, presumably binary
35# translator is not almighty;
36
37$flavour=shift;
38$output=shift;
39
40$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
41( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
42( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
43die "can't locate arm-xlate.pl";
44
45open OUT,"| \"$^X\" $xlate $flavour $output";
46*STDOUT=*OUT;
47
48my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
49my ($mac,$nonce)=($inp,$len);
50
51my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));
52
53$code.=<<___;
54#include "arm_arch.h"
55
56.text
57
58// forward "declarations" are required for Apple
59.extern OPENSSL_armcap_P
60.globl poly1305_blocks
61.globl poly1305_emit
62
63.globl poly1305_init
64.type poly1305_init,%function
65.align 5
66poly1305_init:
67 cmp $inp,xzr
68 stp xzr,xzr,[$ctx] // zero hash value
69 stp xzr,xzr,[$ctx,#16] // [along with is_base2_26]
70
71 csel x0,xzr,x0,eq
72 b.eq .Lno_key
73
74#ifdef __ILP32__
75 ldrsw $t1,.LOPENSSL_armcap_P
76#else
77 ldr $t1,.LOPENSSL_armcap_P
78#endif
79 adr $t0,.LOPENSSL_armcap_P
80
81 ldp $r0,$r1,[$inp] // load key
82 mov $s1,#0xfffffffc0fffffff
83 movk $s1,#0x0fff,lsl#48
84 ldr w17,[$t0,$t1]
85#ifdef __ARMEB__
86 rev $r0,$r0 // flip bytes
87 rev $r1,$r1
88#endif
89 and $r0,$r0,$s1 // &=0ffffffc0fffffff
90 and $s1,$s1,#-4
91 and $r1,$r1,$s1 // &=0ffffffc0ffffffc
92 stp $r0,$r1,[$ctx,#32] // save key value
93
94 tst w17,#ARMV7_NEON
95
96 adr $d0,poly1305_blocks
97 adr $r0,poly1305_blocks_neon
98 adr $d1,poly1305_emit
99 adr $r1,poly1305_emit_neon
100
101 csel $d0,$d0,$r0,eq
102 csel $d1,$d1,$r1,eq
103
104#ifdef __ILP32__
105 stp w12,w13,[$len]
106#else
107 stp $d0,$d1,[$len]
108#endif
109
110 mov x0,#1
111.Lno_key:
112 ret
113.size poly1305_init,.-poly1305_init
114
115.type poly1305_blocks,%function
116.align 5
117poly1305_blocks:
118 ands $len,$len,#-16
119 b.eq .Lno_data
120
121 ldp $h0,$h1,[$ctx] // load hash value
122 ldp $r0,$r1,[$ctx,#32] // load key value
123 ldr $h2,[$ctx,#16]
124 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
125 b .Loop
126
127.align 5
128.Loop:
129 ldp $t0,$t1,[$inp],#16 // load input
130 sub $len,$len,#16
131#ifdef __ARMEB__
132 rev $t0,$t0
133 rev $t1,$t1
134#endif
135 adds $h0,$h0,$t0 // accumulate input
136 adcs $h1,$h1,$t1
137
138 mul $d0,$h0,$r0 // h0*r0
139 adc $h2,$h2,$padbit
140 umulh $d1,$h0,$r0
141
142 mul $t0,$h1,$s1 // h1*5*r1
143 umulh $t1,$h1,$s1
144
145 adds $d0,$d0,$t0
146 mul $t0,$h0,$r1 // h0*r1
147 adc $d1,$d1,$t1
148 umulh $d2,$h0,$r1
149
150 adds $d1,$d1,$t0
151 mul $t0,$h1,$r0 // h1*r0
152 adc $d2,$d2,xzr
153 umulh $t1,$h1,$r0
154
155 adds $d1,$d1,$t0
156 mul $t0,$h2,$s1 // h2*5*r1
157 adc $d2,$d2,$t1
158 mul $t1,$h2,$r0 // h2*r0
159
160 adds $d1,$d1,$t0
161 adc $d2,$d2,$t1
162
163 and $t0,$d2,#-4 // final reduction
164 and $h2,$d2,#3
165 add $t0,$t0,$d2,lsr#2
166 adds $h0,$d0,$t0
167 adcs $h1,$d1,xzr
168 adc $h2,$h2,xzr
169
170 cbnz $len,.Loop
171
172 stp $h0,$h1,[$ctx] // store hash value
173 str $h2,[$ctx,#16]
174
175.Lno_data:
176 ret
177.size poly1305_blocks,.-poly1305_blocks
178
179.type poly1305_emit,%function
180.align 5
181poly1305_emit:
182 ldp $h0,$h1,[$ctx] // load hash base 2^64
183 ldr $h2,[$ctx,#16]
184 ldp $t0,$t1,[$nonce] // load nonce
185
186 adds $d0,$h0,#5 // compare to modulus
187 adcs $d1,$h1,xzr
188 adc $d2,$h2,xzr
189
190 tst $d2,#-4 // see if it's carried/borrowed
191
192 csel $h0,$h0,$d0,eq
193 csel $h1,$h1,$d1,eq
194
195#ifdef __ARMEB__
196 ror $t0,$t0,#32 // flip nonce words
197 ror $t1,$t1,#32
198#endif
199 adds $h0,$h0,$t0 // accumulate nonce
200 adc $h1,$h1,$t1
201#ifdef __ARMEB__
202 rev $h0,$h0 // flip output bytes
203 rev $h1,$h1
204#endif
205 stp $h0,$h1,[$mac] // write result
206
207 ret
208.size poly1305_emit,.-poly1305_emit
209___
210my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
211my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
212my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
213my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
214my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
215my ($T0,$T1,$MASK) = map("v$_",(29..31));
216
217my ($in2,$zeros)=("x16","x17");
218my $is_base2_26 = $zeros; # borrow
219
220$code.=<<___;
221.type poly1305_mult,%function
222.align 5
223poly1305_mult:
224 mul $d0,$h0,$r0 // h0*r0
225 umulh $d1,$h0,$r0
226
227 mul $t0,$h1,$s1 // h1*5*r1
228 umulh $t1,$h1,$s1
229
230 adds $d0,$d0,$t0
231 mul $t0,$h0,$r1 // h0*r1
232 adc $d1,$d1,$t1
233 umulh $d2,$h0,$r1
234
235 adds $d1,$d1,$t0
236 mul $t0,$h1,$r0 // h1*r0
237 adc $d2,$d2,xzr
238 umulh $t1,$h1,$r0
239
240 adds $d1,$d1,$t0
241 mul $t0,$h2,$s1 // h2*5*r1
242 adc $d2,$d2,$t1
243 mul $t1,$h2,$r0 // h2*r0
244
245 adds $d1,$d1,$t0
246 adc $d2,$d2,$t1
247
248 and $t0,$d2,#-4 // final reduction
249 and $h2,$d2,#3
250 add $t0,$t0,$d2,lsr#2
251 adds $h0,$d0,$t0
252 adcs $h1,$d1,xzr
253 adc $h2,$h2,xzr
254
255 ret
256.size poly1305_mult,.-poly1305_mult
257
258.type poly1305_splat,%function
259.align 5
260poly1305_splat:
261 and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26
262 ubfx x13,$h0,#26,#26
263 extr x14,$h1,$h0,#52
264 and x14,x14,#0x03ffffff
265 ubfx x15,$h1,#14,#26
266 extr x16,$h2,$h1,#40
267
268 str w12,[$ctx,#16*0] // r0
269 add w12,w13,w13,lsl#2 // r1*5
270 str w13,[$ctx,#16*1] // r1
271 add w13,w14,w14,lsl#2 // r2*5
272 str w12,[$ctx,#16*2] // s1
273 str w14,[$ctx,#16*3] // r2
274 add w14,w15,w15,lsl#2 // r3*5
275 str w13,[$ctx,#16*4] // s2
276 str w15,[$ctx,#16*5] // r3
277 add w15,w16,w16,lsl#2 // r4*5
278 str w14,[$ctx,#16*6] // s3
279 str w16,[$ctx,#16*7] // r4
280 str w15,[$ctx,#16*8] // s4
281
282 ret
283.size poly1305_splat,.-poly1305_splat
284
285.type poly1305_blocks_neon,%function
286.align 5
287poly1305_blocks_neon:
288 ldr $is_base2_26,[$ctx,#24]
289 cmp $len,#128
290 b.hs .Lblocks_neon
291 cbz $is_base2_26,poly1305_blocks
292
293.Lblocks_neon:
294 .inst 0xd503233f // paciasp
295 stp x29,x30,[sp,#-80]!
296 add x29,sp,#0
297
298 ands $len,$len,#-16
299 b.eq .Lno_data_neon
300
301 cbz $is_base2_26,.Lbase2_64_neon
302
303 ldp w10,w11,[$ctx] // load hash value base 2^26
304 ldp w12,w13,[$ctx,#8]
305 ldr w14,[$ctx,#16]
306
307 tst $len,#31
308 b.eq .Leven_neon
309
310 ldp $r0,$r1,[$ctx,#32] // load key value
311
312 add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
313 lsr $h1,x12,#12
314 adds $h0,$h0,x12,lsl#52
315 add $h1,$h1,x13,lsl#14
316 adc $h1,$h1,xzr
317 lsr $h2,x14,#24
318 adds $h1,$h1,x14,lsl#40
319 adc $d2,$h2,xzr // can be partially reduced...
320
321 ldp $d0,$d1,[$inp],#16 // load input
322 sub $len,$len,#16
323 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
324
325 and $t0,$d2,#-4 // ... so reduce
326 and $h2,$d2,#3
327 add $t0,$t0,$d2,lsr#2
328 adds $h0,$h0,$t0
329 adcs $h1,$h1,xzr
330 adc $h2,$h2,xzr
331
332#ifdef __ARMEB__
333 rev $d0,$d0
334 rev $d1,$d1
335#endif
336 adds $h0,$h0,$d0 // accumulate input
337 adcs $h1,$h1,$d1
338 adc $h2,$h2,$padbit
339
340 bl poly1305_mult
341 ldr x30,[sp,#8]
342
343 cbz $padbit,.Lstore_base2_64_neon
344
345 and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
346 ubfx x11,$h0,#26,#26
347 extr x12,$h1,$h0,#52
348 and x12,x12,#0x03ffffff
349 ubfx x13,$h1,#14,#26
350 extr x14,$h2,$h1,#40
351
352 cbnz $len,.Leven_neon
353
354 stp w10,w11,[$ctx] // store hash value base 2^26
355 stp w12,w13,[$ctx,#8]
356 str w14,[$ctx,#16]
357 b .Lno_data_neon
358
359.align 4
360.Lstore_base2_64_neon:
361 stp $h0,$h1,[$ctx] // store hash value base 2^64
362 stp $h2,xzr,[$ctx,#16] // note that is_base2_26 is zeroed
363 b .Lno_data_neon
364
365.align 4
366.Lbase2_64_neon:
367 ldp $r0,$r1,[$ctx,#32] // load key value
368
369 ldp $h0,$h1,[$ctx] // load hash value base 2^64
370 ldr $h2,[$ctx,#16]
371
372 tst $len,#31
373 b.eq .Linit_neon
374
375 ldp $d0,$d1,[$inp],#16 // load input
376 sub $len,$len,#16
377 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
378#ifdef __ARMEB__
379 rev $d0,$d0
380 rev $d1,$d1
381#endif
382 adds $h0,$h0,$d0 // accumulate input
383 adcs $h1,$h1,$d1
384 adc $h2,$h2,$padbit
385
386 bl poly1305_mult
387
388.Linit_neon:
389 and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
390 ubfx x11,$h0,#26,#26
391 extr x12,$h1,$h0,#52
392 and x12,x12,#0x03ffffff
393 ubfx x13,$h1,#14,#26
394 extr x14,$h2,$h1,#40
395
396 stp d8,d9,[sp,#16] // meet ABI requirements
397 stp d10,d11,[sp,#32]
398 stp d12,d13,[sp,#48]
399 stp d14,d15,[sp,#64]
400
401 fmov ${H0},x10
402 fmov ${H1},x11
403 fmov ${H2},x12
404 fmov ${H3},x13
405 fmov ${H4},x14
406
407 ////////////////////////////////// initialize r^n table
408 mov $h0,$r0 // r^1
409 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
410 mov $h1,$r1
411 mov $h2,xzr
412 add $ctx,$ctx,#48+12
413 bl poly1305_splat
414
415 bl poly1305_mult // r^2
416 sub $ctx,$ctx,#4
417 bl poly1305_splat
418
419 bl poly1305_mult // r^3
420 sub $ctx,$ctx,#4
421 bl poly1305_splat
422
423 bl poly1305_mult // r^4
424 sub $ctx,$ctx,#4
425 bl poly1305_splat
426 ldr x30,[sp,#8]
427
428 add $in2,$inp,#32
429 adr $zeros,.Lzeros
430 subs $len,$len,#64
431 csel $in2,$zeros,$in2,lo
432
433 mov x4,#1
434 str x4,[$ctx,#-24] // set is_base2_26
435 sub $ctx,$ctx,#48 // restore original $ctx
436 b .Ldo_neon
437
438.align 4
439.Leven_neon:
440 add $in2,$inp,#32
441 adr $zeros,.Lzeros
442 subs $len,$len,#64
443 csel $in2,$zeros,$in2,lo
444
445 stp d8,d9,[sp,#16] // meet ABI requirements
446 stp d10,d11,[sp,#32]
447 stp d12,d13,[sp,#48]
448 stp d14,d15,[sp,#64]
449
450 fmov ${H0},x10
451 fmov ${H1},x11
452 fmov ${H2},x12
453 fmov ${H3},x13
454 fmov ${H4},x14
455
456.Ldo_neon:
457 ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
458 ldp x9,x13,[$in2],#48
459
460 lsl $padbit,$padbit,#24
461 add x15,$ctx,#48
462
463#ifdef __ARMEB__
464 rev x8,x8
465 rev x12,x12
466 rev x9,x9
467 rev x13,x13
468#endif
469 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
470 and x5,x9,#0x03ffffff
471 ubfx x6,x8,#26,#26
472 ubfx x7,x9,#26,#26
473 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
474 extr x8,x12,x8,#52
475 extr x9,x13,x9,#52
476 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
477 fmov $IN23_0,x4
478 and x8,x8,#0x03ffffff
479 and x9,x9,#0x03ffffff
480 ubfx x10,x12,#14,#26
481 ubfx x11,x13,#14,#26
482 add x12,$padbit,x12,lsr#40
483 add x13,$padbit,x13,lsr#40
484 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
485 fmov $IN23_1,x6
486 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
487 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
488 fmov $IN23_2,x8
489 fmov $IN23_3,x10
490 fmov $IN23_4,x12
491
492 ldp x8,x12,[$inp],#16 // inp[0:1]
493 ldp x9,x13,[$inp],#48
494
495 ld1 {$R0,$R1,$S1,$R2},[x15],#64
496 ld1 {$S2,$R3,$S3,$R4},[x15],#64
497 ld1 {$S4},[x15]
498
499#ifdef __ARMEB__
500 rev x8,x8
501 rev x12,x12
502 rev x9,x9
503 rev x13,x13
504#endif
505 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
506 and x5,x9,#0x03ffffff
507 ubfx x6,x8,#26,#26
508 ubfx x7,x9,#26,#26
509 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
510 extr x8,x12,x8,#52
511 extr x9,x13,x9,#52
512 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
513 fmov $IN01_0,x4
514 and x8,x8,#0x03ffffff
515 and x9,x9,#0x03ffffff
516 ubfx x10,x12,#14,#26
517 ubfx x11,x13,#14,#26
518 add x12,$padbit,x12,lsr#40
519 add x13,$padbit,x13,lsr#40
520 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
521 fmov $IN01_1,x6
522 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
523 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
524 movi $MASK.2d,#-1
525 fmov $IN01_2,x8
526 fmov $IN01_3,x10
527 fmov $IN01_4,x12
528 ushr $MASK.2d,$MASK.2d,#38
529
530 b.ls .Lskip_loop
531
532.align 4
533.Loop_neon:
534 ////////////////////////////////////////////////////////////////
535 // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
536 // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
537 // \___________________/
538 // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
539 // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
540 // \___________________/ \____________________/
541 //
542 // Note that we start with inp[2:3]*r^2. This is because it
543 // doesn't depend on reduction in previous iteration.
544 ////////////////////////////////////////////////////////////////
545 // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0
546 // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4
547 // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3
548 // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2
549 // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
550
551 subs $len,$len,#64
552 umull $ACC4,$IN23_0,${R4}[2]
553 csel $in2,$zeros,$in2,lo
554 umull $ACC3,$IN23_0,${R3}[2]
555 umull $ACC2,$IN23_0,${R2}[2]
556 ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
557 umull $ACC1,$IN23_0,${R1}[2]
558 ldp x9,x13,[$in2],#48
559 umull $ACC0,$IN23_0,${R0}[2]
560#ifdef __ARMEB__
561 rev x8,x8
562 rev x12,x12
563 rev x9,x9
564 rev x13,x13
565#endif
566
567 umlal $ACC4,$IN23_1,${R3}[2]
568 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
569 umlal $ACC3,$IN23_1,${R2}[2]
570 and x5,x9,#0x03ffffff
571 umlal $ACC2,$IN23_1,${R1}[2]
572 ubfx x6,x8,#26,#26
573 umlal $ACC1,$IN23_1,${R0}[2]
574 ubfx x7,x9,#26,#26
575 umlal $ACC0,$IN23_1,${S4}[2]
576 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
577
578 umlal $ACC4,$IN23_2,${R2}[2]
579 extr x8,x12,x8,#52
580 umlal $ACC3,$IN23_2,${R1}[2]
581 extr x9,x13,x9,#52
582 umlal $ACC2,$IN23_2,${R0}[2]
583 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
584 umlal $ACC1,$IN23_2,${S4}[2]
585 fmov $IN23_0,x4
586 umlal $ACC0,$IN23_2,${S3}[2]
587 and x8,x8,#0x03ffffff
588
589 umlal $ACC4,$IN23_3,${R1}[2]
590 and x9,x9,#0x03ffffff
591 umlal $ACC3,$IN23_3,${R0}[2]
592 ubfx x10,x12,#14,#26
593 umlal $ACC2,$IN23_3,${S4}[2]
594 ubfx x11,x13,#14,#26
595 umlal $ACC1,$IN23_3,${S3}[2]
596 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
597 umlal $ACC0,$IN23_3,${S2}[2]
598 fmov $IN23_1,x6
599
600 add $IN01_2,$IN01_2,$H2
601 add x12,$padbit,x12,lsr#40
602 umlal $ACC4,$IN23_4,${R0}[2]
603 add x13,$padbit,x13,lsr#40
604 umlal $ACC3,$IN23_4,${S4}[2]
605 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
606 umlal $ACC2,$IN23_4,${S3}[2]
607 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
608 umlal $ACC1,$IN23_4,${S2}[2]
609 fmov $IN23_2,x8
610 umlal $ACC0,$IN23_4,${S1}[2]
611 fmov $IN23_3,x10
612
613 ////////////////////////////////////////////////////////////////
614 // (hash+inp[0:1])*r^4 and accumulate
615
616 add $IN01_0,$IN01_0,$H0
617 fmov $IN23_4,x12
618 umlal $ACC3,$IN01_2,${R1}[0]
619 ldp x8,x12,[$inp],#16 // inp[0:1]
620 umlal $ACC0,$IN01_2,${S3}[0]
621 ldp x9,x13,[$inp],#48
622 umlal $ACC4,$IN01_2,${R2}[0]
623 umlal $ACC1,$IN01_2,${S4}[0]
624 umlal $ACC2,$IN01_2,${R0}[0]
625#ifdef __ARMEB__
626 rev x8,x8
627 rev x12,x12
628 rev x9,x9
629 rev x13,x13
630#endif
631
632 add $IN01_1,$IN01_1,$H1
633 umlal $ACC3,$IN01_0,${R3}[0]
634 umlal $ACC4,$IN01_0,${R4}[0]
635 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
636 umlal $ACC2,$IN01_0,${R2}[0]
637 and x5,x9,#0x03ffffff
638 umlal $ACC0,$IN01_0,${R0}[0]
639 ubfx x6,x8,#26,#26
640 umlal $ACC1,$IN01_0,${R1}[0]
641 ubfx x7,x9,#26,#26
642
643 add $IN01_3,$IN01_3,$H3
644 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
645 umlal $ACC3,$IN01_1,${R2}[0]
646 extr x8,x12,x8,#52
647 umlal $ACC4,$IN01_1,${R3}[0]
648 extr x9,x13,x9,#52
649 umlal $ACC0,$IN01_1,${S4}[0]
650 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
651 umlal $ACC2,$IN01_1,${R1}[0]
652 fmov $IN01_0,x4
653 umlal $ACC1,$IN01_1,${R0}[0]
654 and x8,x8,#0x03ffffff
655
656 add $IN01_4,$IN01_4,$H4
657 and x9,x9,#0x03ffffff
658 umlal $ACC3,$IN01_3,${R0}[0]
659 ubfx x10,x12,#14,#26
660 umlal $ACC0,$IN01_3,${S2}[0]
661 ubfx x11,x13,#14,#26
662 umlal $ACC4,$IN01_3,${R1}[0]
663 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
664 umlal $ACC1,$IN01_3,${S3}[0]
665 fmov $IN01_1,x6
666 umlal $ACC2,$IN01_3,${S4}[0]
667 add x12,$padbit,x12,lsr#40
668
669 umlal $ACC3,$IN01_4,${S4}[0]
670 add x13,$padbit,x13,lsr#40
671 umlal $ACC0,$IN01_4,${S1}[0]
672 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
673 umlal $ACC4,$IN01_4,${R0}[0]
674 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
675 umlal $ACC1,$IN01_4,${S2}[0]
676 fmov $IN01_2,x8
677 umlal $ACC2,$IN01_4,${S3}[0]
678 fmov $IN01_3,x10
679 fmov $IN01_4,x12
680
681 /////////////////////////////////////////////////////////////////
682 // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
683 // and P. Schwabe
684 //
685 // [see discussion in poly1305-armv4 module]
686
687 ushr $T0.2d,$ACC3,#26
688 xtn $H3,$ACC3
689 ushr $T1.2d,$ACC0,#26
690 and $ACC0,$ACC0,$MASK.2d
691 add $ACC4,$ACC4,$T0.2d // h3 -> h4
692 bic $H3,#0xfc,lsl#24 // &=0x03ffffff
693 add $ACC1,$ACC1,$T1.2d // h0 -> h1
694
695 ushr $T0.2d,$ACC4,#26
696 xtn $H4,$ACC4
697 ushr $T1.2d,$ACC1,#26
698 xtn $H1,$ACC1
699 bic $H4,#0xfc,lsl#24
700 add $ACC2,$ACC2,$T1.2d // h1 -> h2
701
702 add $ACC0,$ACC0,$T0.2d
703 shl $T0.2d,$T0.2d,#2
704 shrn $T1.2s,$ACC2,#26
705 xtn $H2,$ACC2
706 add $ACC0,$ACC0,$T0.2d // h4 -> h0
707 bic $H1,#0xfc,lsl#24
708 add $H3,$H3,$T1.2s // h2 -> h3
709 bic $H2,#0xfc,lsl#24
710
711 shrn $T0.2s,$ACC0,#26
712 xtn $H0,$ACC0
713 ushr $T1.2s,$H3,#26
714 bic $H3,#0xfc,lsl#24
715 bic $H0,#0xfc,lsl#24
716 add $H1,$H1,$T0.2s // h0 -> h1
717 add $H4,$H4,$T1.2s // h3 -> h4
718
719 b.hi .Loop_neon
720
721.Lskip_loop:
722 dup $IN23_2,${IN23_2}[0]
723 add $IN01_2,$IN01_2,$H2
724
725 ////////////////////////////////////////////////////////////////
726 // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
727
728 adds $len,$len,#32
729 b.ne .Long_tail
730
731 dup $IN23_2,${IN01_2}[0]
732 add $IN23_0,$IN01_0,$H0
733 add $IN23_3,$IN01_3,$H3
734 add $IN23_1,$IN01_1,$H1
735 add $IN23_4,$IN01_4,$H4
736
737.Long_tail:
738 dup $IN23_0,${IN23_0}[0]
739 umull2 $ACC0,$IN23_2,${S3}
740 umull2 $ACC3,$IN23_2,${R1}
741 umull2 $ACC4,$IN23_2,${R2}
742 umull2 $ACC2,$IN23_2,${R0}
743 umull2 $ACC1,$IN23_2,${S4}
744
745 dup $IN23_1,${IN23_1}[0]
746 umlal2 $ACC0,$IN23_0,${R0}
747 umlal2 $ACC2,$IN23_0,${R2}
748 umlal2 $ACC3,$IN23_0,${R3}
749 umlal2 $ACC4,$IN23_0,${R4}
750 umlal2 $ACC1,$IN23_0,${R1}
751
752 dup $IN23_3,${IN23_3}[0]
753 umlal2 $ACC0,$IN23_1,${S4}
754 umlal2 $ACC3,$IN23_1,${R2}
755 umlal2 $ACC2,$IN23_1,${R1}
756 umlal2 $ACC4,$IN23_1,${R3}
757 umlal2 $ACC1,$IN23_1,${R0}
758
759 dup $IN23_4,${IN23_4}[0]
760 umlal2 $ACC3,$IN23_3,${R0}
761 umlal2 $ACC4,$IN23_3,${R1}
762 umlal2 $ACC0,$IN23_3,${S2}
763 umlal2 $ACC1,$IN23_3,${S3}
764 umlal2 $ACC2,$IN23_3,${S4}
765
766 umlal2 $ACC3,$IN23_4,${S4}
767 umlal2 $ACC0,$IN23_4,${S1}
768 umlal2 $ACC4,$IN23_4,${R0}
769 umlal2 $ACC1,$IN23_4,${S2}
770 umlal2 $ACC2,$IN23_4,${S3}
771
772 b.eq .Lshort_tail
773
774 ////////////////////////////////////////////////////////////////
775 // (hash+inp[0:1])*r^4:r^3 and accumulate
776
777 add $IN01_0,$IN01_0,$H0
778 umlal $ACC3,$IN01_2,${R1}
779 umlal $ACC0,$IN01_2,${S3}
780 umlal $ACC4,$IN01_2,${R2}
781 umlal $ACC1,$IN01_2,${S4}
782 umlal $ACC2,$IN01_2,${R0}
783
784 add $IN01_1,$IN01_1,$H1
785 umlal $ACC3,$IN01_0,${R3}
786 umlal $ACC0,$IN01_0,${R0}
787 umlal $ACC4,$IN01_0,${R4}
788 umlal $ACC1,$IN01_0,${R1}
789 umlal $ACC2,$IN01_0,${R2}
790
791 add $IN01_3,$IN01_3,$H3
792 umlal $ACC3,$IN01_1,${R2}
793 umlal $ACC0,$IN01_1,${S4}
794 umlal $ACC4,$IN01_1,${R3}
795 umlal $ACC1,$IN01_1,${R0}
796 umlal $ACC2,$IN01_1,${R1}
797
798 add $IN01_4,$IN01_4,$H4
799 umlal $ACC3,$IN01_3,${R0}
800 umlal $ACC0,$IN01_3,${S2}
801 umlal $ACC4,$IN01_3,${R1}
802 umlal $ACC1,$IN01_3,${S3}
803 umlal $ACC2,$IN01_3,${S4}
804
805 umlal $ACC3,$IN01_4,${S4}
806 umlal $ACC0,$IN01_4,${S1}
807 umlal $ACC4,$IN01_4,${R0}
808 umlal $ACC1,$IN01_4,${S2}
809 umlal $ACC2,$IN01_4,${S3}
810
811.Lshort_tail:
812 ////////////////////////////////////////////////////////////////
813 // horizontal add
814
815 addp $ACC3,$ACC3,$ACC3
816 ldp d8,d9,[sp,#16] // meet ABI requirements
817 addp $ACC0,$ACC0,$ACC0
818 ldp d10,d11,[sp,#32]
819 addp $ACC4,$ACC4,$ACC4
820 ldp d12,d13,[sp,#48]
821 addp $ACC1,$ACC1,$ACC1
822 ldp d14,d15,[sp,#64]
823 addp $ACC2,$ACC2,$ACC2
824
825 ////////////////////////////////////////////////////////////////
826 // lazy reduction, but without narrowing
827
828 ushr $T0.2d,$ACC3,#26
829 and $ACC3,$ACC3,$MASK.2d
830 ushr $T1.2d,$ACC0,#26
831 and $ACC0,$ACC0,$MASK.2d
832
833 add $ACC4,$ACC4,$T0.2d // h3 -> h4
834 add $ACC1,$ACC1,$T1.2d // h0 -> h1
835
836 ushr $T0.2d,$ACC4,#26
837 and $ACC4,$ACC4,$MASK.2d
838 ushr $T1.2d,$ACC1,#26
839 and $ACC1,$ACC1,$MASK.2d
840 add $ACC2,$ACC2,$T1.2d // h1 -> h2
841
842 add $ACC0,$ACC0,$T0.2d
843 shl $T0.2d,$T0.2d,#2
844 ushr $T1.2d,$ACC2,#26
845 and $ACC2,$ACC2,$MASK.2d
846 add $ACC0,$ACC0,$T0.2d // h4 -> h0
847 add $ACC3,$ACC3,$T1.2d // h2 -> h3
848
849 ushr $T0.2d,$ACC0,#26
850 and $ACC0,$ACC0,$MASK.2d
851 ushr $T1.2d,$ACC3,#26
852 and $ACC3,$ACC3,$MASK.2d
853 add $ACC1,$ACC1,$T0.2d // h0 -> h1
854 add $ACC4,$ACC4,$T1.2d // h3 -> h4
855
856 ////////////////////////////////////////////////////////////////
857 // write the result, can be partially reduced
858
859 st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
860 st1 {$ACC4}[0],[$ctx]
861
862.Lno_data_neon:
863 .inst 0xd50323bf // autiasp
864 ldr x29,[sp],#80
865 ret
866.size poly1305_blocks_neon,.-poly1305_blocks_neon
867
868.type poly1305_emit_neon,%function
869.align 5
870poly1305_emit_neon:
871 ldr $is_base2_26,[$ctx,#24]
872 cbz $is_base2_26,poly1305_emit
873
874 ldp w10,w11,[$ctx] // load hash value base 2^26
875 ldp w12,w13,[$ctx,#8]
876 ldr w14,[$ctx,#16]
877
878 add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
879 lsr $h1,x12,#12
880 adds $h0,$h0,x12,lsl#52
881 add $h1,$h1,x13,lsl#14
882 adc $h1,$h1,xzr
883 lsr $h2,x14,#24
884 adds $h1,$h1,x14,lsl#40
885 adc $h2,$h2,xzr // can be partially reduced...
886
887 ldp $t0,$t1,[$nonce] // load nonce
888
889 and $d0,$h2,#-4 // ... so reduce
890 add $d0,$d0,$h2,lsr#2
891 and $h2,$h2,#3
892 adds $h0,$h0,$d0
893 adcs $h1,$h1,xzr
894 adc $h2,$h2,xzr
895
896 adds $d0,$h0,#5 // compare to modulus
897 adcs $d1,$h1,xzr
898 adc $d2,$h2,xzr
899
900 tst $d2,#-4 // see if it's carried/borrowed
901
902 csel $h0,$h0,$d0,eq
903 csel $h1,$h1,$d1,eq
904
905#ifdef __ARMEB__
906 ror $t0,$t0,#32 // flip nonce words
907 ror $t1,$t1,#32
908#endif
909 adds $h0,$h0,$t0 // accumulate nonce
910 adc $h1,$h1,$t1
911#ifdef __ARMEB__
912 rev $h0,$h0 // flip output bytes
913 rev $h1,$h1
914#endif
915 stp $h0,$h1,[$mac] // write result
916
917 ret
918.size poly1305_emit_neon,.-poly1305_emit_neon
919
920.align 5
921.Lzeros:
922.long 0,0,0,0,0,0,0,0
923.LOPENSSL_armcap_P:
924#ifdef __ILP32__
925.long OPENSSL_armcap_P-.
926#else
927.quad OPENSSL_armcap_P-.
928#endif
929.asciz "Poly1305 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
930.align 2
931___
932
933foreach (split("\n",$code)) {
934 s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or
935 s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or
936 (m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or
937 (m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or
938 (m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or
939 (m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or
940 (m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));
941
942 s/\.[124]([sd])\[/.$1\[/;
943
944 print $_,"\n";
945}
946close STDOUT or die "error closing STDOUT: $!";
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette