ecp_nistz256-ppc64.pl@ 90293

Last change on this file since 90293 was 90293, checked in by vboxsync, 4 years ago
openssl-1.1.1k: Applied and adjusted our OpenSSL changes to 1.1.1k. bugref:10072
File size: 51.5 KB

Line
1	#! /usr/bin/env perl
2	# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
3	#
4	# Licensed under the OpenSSL license (the "License"). You may not use
5	# this file except in compliance with the License. You can obtain a copy
6	# in the file LICENSE in the source distribution or at
7	# https://www.openssl.org/source/license.html
8
9	#
10	# ====================================================================
11	# Written by Andy Polyakov <[email protected]> for the OpenSSL
12	# project. The module is, however, dual licensed under OpenSSL and
13	# CRYPTOGAMS licenses depending on where you obtain it. For further
14	# details see http://www.openssl.org/~appro/cryptogams/.
15	# ====================================================================
16	#
17	# ECP_NISTZ256 module for PPC64.
18	#
19	# August 2016.
20	#
21	# Original ECP_NISTZ256 submission targeting x86_64 is detailed in
22	# http://eprint.iacr.org/2013/816.
23	#
24	# with/without -DECP_NISTZ256_ASM
25	# POWER7 +260-530%
26	# POWER8 +220-340%
27
28	$flavour = shift;
29	while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
30
31	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
32	( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
33	( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
34	die "can't locate ppc-xlate.pl";
35
36	open OUT,"\| \"$^X\" $xlate $flavour $output";
37	STDOUT=OUT;
38
39	my $sp="r1";
40
41	{
42	my ($rp,$ap,$bp,$bi,$acc0,$acc1,$acc2,$acc3,$poly1,$poly3,
43	$acc4,$acc5,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3) =
44	map("r$_",(3..12,22..31));
45
46	my ($acc6,$acc7)=($bp,$bi); # used in __ecp_nistz256_sqr_mont
47
48	$code.=<<___;
49	.machine "any"
50	.text
51	___
52	########################################################################
53	# Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
54	#
55	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
56	open TABLE,"<ecp_nistz256_table.c" or
57	open TABLE,"<${dir}../ecp_nistz256_table.c" or
58	die "failed to open ecp_nistz256_table.c:",$!;
59
60	use integer;
61
62	foreach(<TABLE>) {
63	s/TOBN$\s(0x[0-9a-f]+),\s(0x[0-9a-f]+)\s*$/push @arr,hex($2),hex($1)/geo;
64	}
65	close TABLE;
66
67	# See ecp_nistz256_table.c for explanation for why it's 641637.
68	# 641637-1 is because $#arr returns last valid index or @arr, not
69	# amount of elements.
70	die "insane number of elements" if ($#arr != 641637-1);
71
72	$code.=<<___;
73	.type ecp_nistz256_precomputed,\@object
74	.globl ecp_nistz256_precomputed
75	.align 12
76	ecp_nistz256_precomputed:
77	___
78	########################################################################
79	# this conversion smashes P256_POINT_AFFINE by individual bytes with
80	# 64 byte interval, similar to
81	# 1111222233334444
82	# 1234123412341234
83	for(1..37) {
84	@tbl = splice(@arr,0,64*16);
85	for($i=0;$i<64;$i++) {
86	undef @line;
87	for($j=0;$j<64;$j++) {
88	push @line,(@tbl[$j16+$i/4]>>(($i%4)8))&0xff;
89	}
90	$code.=".byte\t";
91	$code.=join(',',map { sprintf "0x%02x",$_} @line);
92	$code.="\n";
93	}
94	}
95
96	$code.=<<___;
97	.size ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
98	.asciz "ECP_NISTZ256 for PPC64, CRYPTOGAMS by <appro\@openssl.org>"
99
100	# void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4],
101	# const BN_ULONG x2[4]);
102	.globl ecp_nistz256_mul_mont
103	.align 5
104	ecp_nistz256_mul_mont:
105	stdu $sp,-128($sp)
106	mflr r0
107	std r22,48($sp)
108	std r23,56($sp)
109	std r24,64($sp)
110	std r25,72($sp)
111	std r26,80($sp)
112	std r27,88($sp)
113	std r28,96($sp)
114	std r29,104($sp)
115	std r30,112($sp)
116	std r31,120($sp)
117
118	ld $a0,0($ap)
119	ld $bi,0($bp)
120	ld $a1,8($ap)
121	ld $a2,16($ap)
122	ld $a3,24($ap)
123
124	li $poly1,-1
125	srdi $poly1,$poly1,32 # 0x00000000ffffffff
126	li $poly3,1
127	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
128
129	bl __ecp_nistz256_mul_mont
130
131	mtlr r0
132	ld r22,48($sp)
133	ld r23,56($sp)
134	ld r24,64($sp)
135	ld r25,72($sp)
136	ld r26,80($sp)
137	ld r27,88($sp)
138	ld r28,96($sp)
139	ld r29,104($sp)
140	ld r30,112($sp)
141	ld r31,120($sp)
142	addi $sp,$sp,128
143	blr
144	.long 0
145	.byte 0,12,4,0,0x80,10,3,0
146	.long 0
147	.size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
148
149	# void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
150	.globl ecp_nistz256_sqr_mont
151	.align 4
152	ecp_nistz256_sqr_mont:
153	stdu $sp,-128($sp)
154	mflr r0
155	std r22,48($sp)
156	std r23,56($sp)
157	std r24,64($sp)
158	std r25,72($sp)
159	std r26,80($sp)
160	std r27,88($sp)
161	std r28,96($sp)
162	std r29,104($sp)
163	std r30,112($sp)
164	std r31,120($sp)
165
166	ld $a0,0($ap)
167	ld $a1,8($ap)
168	ld $a2,16($ap)
169	ld $a3,24($ap)
170
171	li $poly1,-1
172	srdi $poly1,$poly1,32 # 0x00000000ffffffff
173	li $poly3,1
174	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
175
176	bl __ecp_nistz256_sqr_mont
177
178	mtlr r0
179	ld r22,48($sp)
180	ld r23,56($sp)
181	ld r24,64($sp)
182	ld r25,72($sp)
183	ld r26,80($sp)
184	ld r27,88($sp)
185	ld r28,96($sp)
186	ld r29,104($sp)
187	ld r30,112($sp)
188	ld r31,120($sp)
189	addi $sp,$sp,128
190	blr
191	.long 0
192	.byte 0,12,4,0,0x80,10,2,0
193	.long 0
194	.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
195
196	# void ecp_nistz256_add(BN_ULONG x0[4],const BN_ULONG x1[4],
197	# const BN_ULONG x2[4]);
198	.globl ecp_nistz256_add
199	.align 4
200	ecp_nistz256_add:
201	stdu $sp,-128($sp)
202	mflr r0
203	std r28,96($sp)
204	std r29,104($sp)
205	std r30,112($sp)
206	std r31,120($sp)
207
208	ld $acc0,0($ap)
209	ld $t0, 0($bp)
210	ld $acc1,8($ap)
211	ld $t1, 8($bp)
212	ld $acc2,16($ap)
213	ld $t2, 16($bp)
214	ld $acc3,24($ap)
215	ld $t3, 24($bp)
216
217	li $poly1,-1
218	srdi $poly1,$poly1,32 # 0x00000000ffffffff
219	li $poly3,1
220	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
221
222	bl __ecp_nistz256_add
223
224	mtlr r0
225	ld r28,96($sp)
226	ld r29,104($sp)
227	ld r30,112($sp)
228	ld r31,120($sp)
229	addi $sp,$sp,128
230	blr
231	.long 0
232	.byte 0,12,4,0,0x80,4,3,0
233	.long 0
234	.size ecp_nistz256_add,.-ecp_nistz256_add
235
236	# void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
237	.globl ecp_nistz256_div_by_2
238	.align 4
239	ecp_nistz256_div_by_2:
240	stdu $sp,-128($sp)
241	mflr r0
242	std r28,96($sp)
243	std r29,104($sp)
244	std r30,112($sp)
245	std r31,120($sp)
246
247	ld $acc0,0($ap)
248	ld $acc1,8($ap)
249	ld $acc2,16($ap)
250	ld $acc3,24($ap)
251
252	li $poly1,-1
253	srdi $poly1,$poly1,32 # 0x00000000ffffffff
254	li $poly3,1
255	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
256
257	bl __ecp_nistz256_div_by_2
258
259	mtlr r0
260	ld r28,96($sp)
261	ld r29,104($sp)
262	ld r30,112($sp)
263	ld r31,120($sp)
264	addi $sp,$sp,128
265	blr
266	.long 0
267	.byte 0,12,4,0,0x80,4,2,0
268	.long 0
269	.size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
270
271	# void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
272	.globl ecp_nistz256_mul_by_2
273	.align 4
274	ecp_nistz256_mul_by_2:
275	stdu $sp,-128($sp)
276	mflr r0
277	std r28,96($sp)
278	std r29,104($sp)
279	std r30,112($sp)
280	std r31,120($sp)
281
282	ld $acc0,0($ap)
283	ld $acc1,8($ap)
284	ld $acc2,16($ap)
285	ld $acc3,24($ap)
286
287	mr $t0,$acc0
288	mr $t1,$acc1
289	mr $t2,$acc2
290	mr $t3,$acc3
291
292	li $poly1,-1
293	srdi $poly1,$poly1,32 # 0x00000000ffffffff
294	li $poly3,1
295	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
296
297	bl __ecp_nistz256_add # ret = a+a // 2*a
298
299	mtlr r0
300	ld r28,96($sp)
301	ld r29,104($sp)
302	ld r30,112($sp)
303	ld r31,120($sp)
304	addi $sp,$sp,128
305	blr
306	.long 0
307	.byte 0,12,4,0,0x80,4,3,0
308	.long 0
309	.size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
310
311	# void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]);
312	.globl ecp_nistz256_mul_by_3
313	.align 4
314	ecp_nistz256_mul_by_3:
315	stdu $sp,-128($sp)
316	mflr r0
317	std r28,96($sp)
318	std r29,104($sp)
319	std r30,112($sp)
320	std r31,120($sp)
321
322	ld $acc0,0($ap)
323	ld $acc1,8($ap)
324	ld $acc2,16($ap)
325	ld $acc3,24($ap)
326
327	mr $t0,$acc0
328	std $acc0,64($sp)
329	mr $t1,$acc1
330	std $acc1,72($sp)
331	mr $t2,$acc2
332	std $acc2,80($sp)
333	mr $t3,$acc3
334	std $acc3,88($sp)
335
336	li $poly1,-1
337	srdi $poly1,$poly1,32 # 0x00000000ffffffff
338	li $poly3,1
339	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
340
341	bl __ecp_nistz256_add # ret = a+a // 2*a
342
343	ld $t0,64($sp)
344	ld $t1,72($sp)
345	ld $t2,80($sp)
346	ld $t3,88($sp)
347
348	bl __ecp_nistz256_add # ret += a // 2a+a=3a
349
350	mtlr r0
351	ld r28,96($sp)
352	ld r29,104($sp)
353	ld r30,112($sp)
354	ld r31,120($sp)
355	addi $sp,$sp,128
356	blr
357	.long 0
358	.byte 0,12,4,0,0x80,4,2,0
359	.long 0
360	.size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
361
362	# void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4],
363	# const BN_ULONG x2[4]);
364	.globl ecp_nistz256_sub
365	.align 4
366	ecp_nistz256_sub:
367	stdu $sp,-128($sp)
368	mflr r0
369	std r28,96($sp)
370	std r29,104($sp)
371	std r30,112($sp)
372	std r31,120($sp)
373
374	ld $acc0,0($ap)
375	ld $acc1,8($ap)
376	ld $acc2,16($ap)
377	ld $acc3,24($ap)
378
379	li $poly1,-1
380	srdi $poly1,$poly1,32 # 0x00000000ffffffff
381	li $poly3,1
382	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
383
384	bl __ecp_nistz256_sub_from
385
386	mtlr r0
387	ld r28,96($sp)
388	ld r29,104($sp)
389	ld r30,112($sp)
390	ld r31,120($sp)
391	addi $sp,$sp,128
392	blr
393	.long 0
394	.byte 0,12,4,0,0x80,4,3,0
395	.long 0
396	.size ecp_nistz256_sub,.-ecp_nistz256_sub
397
398	# void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]);
399	.globl ecp_nistz256_neg
400	.align 4
401	ecp_nistz256_neg:
402	stdu $sp,-128($sp)
403	mflr r0
404	std r28,96($sp)
405	std r29,104($sp)
406	std r30,112($sp)
407	std r31,120($sp)
408
409	mr $bp,$ap
410	li $acc0,0
411	li $acc1,0
412	li $acc2,0
413	li $acc3,0
414
415	li $poly1,-1
416	srdi $poly1,$poly1,32 # 0x00000000ffffffff
417	li $poly3,1
418	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
419
420	bl __ecp_nistz256_sub_from
421
422	mtlr r0
423	ld r28,96($sp)
424	ld r29,104($sp)
425	ld r30,112($sp)
426	ld r31,120($sp)
427	addi $sp,$sp,128
428	blr
429	.long 0
430	.byte 0,12,4,0,0x80,4,2,0
431	.long 0
432	.size ecp_nistz256_neg,.-ecp_nistz256_neg
433
434	# note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded
435	# to $a0-$a3 and b[0] - to $bi
436	.type __ecp_nistz256_mul_mont,\@function
437	.align 4
438	__ecp_nistz256_mul_mont:
439	mulld $acc0,$a0,$bi # a[0]*b[0]
440	mulhdu $t0,$a0,$bi
441
442	mulld $acc1,$a1,$bi # a[1]*b[0]
443	mulhdu $t1,$a1,$bi
444
445	mulld $acc2,$a2,$bi # a[2]*b[0]
446	mulhdu $t2,$a2,$bi
447
448	mulld $acc3,$a3,$bi # a[3]*b[0]
449	mulhdu $t3,$a3,$bi
450	ld $bi,8($bp) # b[1]
451
452	addc $acc1,$acc1,$t0 # accumulate high parts of multiplication
453	sldi $t0,$acc0,32
454	adde $acc2,$acc2,$t1
455	srdi $t1,$acc0,32
456	adde $acc3,$acc3,$t2
457	addze $acc4,$t3
458	li $acc5,0
459	___
460	for($i=1;$i<4;$i++) {
461	################################################################
462	# Reduction iteration is normally performed by accumulating
463	# result of multiplication of modulus by "magic" digit [and
464	# omitting least significant word, which is guaranteed to
465	# be 0], but thanks to special form of modulus and "magic"
466	# digit being equal to least significant word, it can be
467	# performed with additions and subtractions alone. Indeed:
468	#
469	# ffff0001.00000000.0000ffff.ffffffff
470	# * abcdefgh
471	# + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
472	#
473	# Now observing that ff..ffx = (2^n-1)x = 2^n*x-x, we
474	# rewrite above as:
475	#
476	# xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
477	# + abcdefgh.abcdefgh.0000abcd.efgh0000.00000000
478	# - 0000abcd.efgh0000.00000000.00000000.abcdefgh
479	#
480	# or marking redundant operations:
481	#
482	# xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.--------
483	# + abcdefgh.abcdefgh.0000abcd.efgh0000.--------
484	# - 0000abcd.efgh0000.--------.--------.--------
485
486	$code.=<<___;
487	subfc $t2,$t0,$acc0 # "*0xffff0001"
488	subfe $t3,$t1,$acc0
489	addc $acc0,$acc1,$t0 # +=acc[0]<<96 and omit acc[0]
490	adde $acc1,$acc2,$t1
491	adde $acc2,$acc3,$t2 # +=acc[0]*0xffff0001
492	adde $acc3,$acc4,$t3
493	addze $acc4,$acc5
494
495	mulld $t0,$a0,$bi # lo(a[0]*b[i])
496	mulld $t1,$a1,$bi # lo(a[1]*b[i])
497	mulld $t2,$a2,$bi # lo(a[2]*b[i])
498	mulld $t3,$a3,$bi # lo(a[3]*b[i])
499	addc $acc0,$acc0,$t0 # accumulate low parts of multiplication
500	mulhdu $t0,$a0,$bi # hi(a[0]*b[i])
501	adde $acc1,$acc1,$t1
502	mulhdu $t1,$a1,$bi # hi(a[1]*b[i])
503	adde $acc2,$acc2,$t2
504	mulhdu $t2,$a2,$bi # hi(a[2]*b[i])
505	adde $acc3,$acc3,$t3
506	mulhdu $t3,$a3,$bi # hi(a[3]*b[i])
507	addze $acc4,$acc4
508	___
509	$code.=<<___ if ($i<3);
510	ld $bi,8*($i+1)($bp) # b[$i+1]
511	___
512	$code.=<<___;
513	addc $acc1,$acc1,$t0 # accumulate high parts of multiplication
514	sldi $t0,$acc0,32
515	adde $acc2,$acc2,$t1
516	srdi $t1,$acc0,32
517	adde $acc3,$acc3,$t2
518	adde $acc4,$acc4,$t3
519	li $acc5,0
520	addze $acc5,$acc5
521	___
522	}
523	$code.=<<___;
524	# last reduction
525	subfc $t2,$t0,$acc0 # "*0xffff0001"
526	subfe $t3,$t1,$acc0
527	addc $acc0,$acc1,$t0 # +=acc[0]<<96 and omit acc[0]
528	adde $acc1,$acc2,$t1
529	adde $acc2,$acc3,$t2 # +=acc[0]*0xffff0001
530	adde $acc3,$acc4,$t3
531	addze $acc4,$acc5
532
533	li $t2,0
534	addic $acc0,$acc0,1 # ret -= modulus
535	subfe $acc1,$poly1,$acc1
536	subfe $acc2,$t2,$acc2
537	subfe $acc3,$poly3,$acc3
538	subfe $acc4,$t2,$acc4
539
540	addc $acc0,$acc0,$acc4 # ret += modulus if borrow
541	and $t1,$poly1,$acc4
542	and $t3,$poly3,$acc4
543	adde $acc1,$acc1,$t1
544	addze $acc2,$acc2
545	adde $acc3,$acc3,$t3
546
547	std $acc0,0($rp)
548	std $acc1,8($rp)
549	std $acc2,16($rp)
550	std $acc3,24($rp)
551
552	blr
553	.long 0
554	.byte 0,12,0x14,0,0,0,1,0
555	.long 0
556	.size __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont
557
558	# note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded
559	# to $a0-$a3
560	.type __ecp_nistz256_sqr_mont,\@function
561	.align 4
562	__ecp_nistz256_sqr_mont:
563	################################################################
564	# \| \| \| \| \| \|a1*a0\| \|
565	# \| \| \| \| \|a2*a0\| \| \|
566	# \| \|a3a2\|a3a0\| \| \| \|
567	# \| \| \| \|a2*a1\| \| \| \|
568	# \| \| \|a3*a1\| \| \| \| \|
569	# *\| \| \| \| \| \| \| \| 2\|
570	# +\|a3a3\|a2a2\|a1a1\|a0a0\|
571	# \|--+--+--+--+--+--+--+--\|
572	# \|A7\|A6\|A5\|A4\|A3\|A2\|A1\|A0\|, where Ax is $accx, i.e. follow $accx
573	#
574	# "can't overflow" below mark carrying into high part of
575	# multiplication result, which can't overflow, because it
576	# can never be all ones.
577
578	mulld $acc1,$a1,$a0 # a[1]*a[0]
579	mulhdu $t1,$a1,$a0
580	mulld $acc2,$a2,$a0 # a[2]*a[0]
581	mulhdu $t2,$a2,$a0
582	mulld $acc3,$a3,$a0 # a[3]*a[0]
583	mulhdu $acc4,$a3,$a0
584
585	addc $acc2,$acc2,$t1 # accumulate high parts of multiplication
586	mulld $t0,$a2,$a1 # a[2]*a[1]
587	mulhdu $t1,$a2,$a1
588	adde $acc3,$acc3,$t2
589	mulld $t2,$a3,$a1 # a[3]*a[1]
590	mulhdu $t3,$a3,$a1
591	addze $acc4,$acc4 # can't overflow
592
593	mulld $acc5,$a3,$a2 # a[3]*a[2]
594	mulhdu $acc6,$a3,$a2
595
596	addc $t1,$t1,$t2 # accumulate high parts of multiplication
597	addze $t2,$t3 # can't overflow
598
599	addc $acc3,$acc3,$t0 # accumulate low parts of multiplication
600	adde $acc4,$acc4,$t1
601	adde $acc5,$acc5,$t2
602	addze $acc6,$acc6 # can't overflow
603
604	addc $acc1,$acc1,$acc1 # acc[1-6]*=2
605	adde $acc2,$acc2,$acc2
606	adde $acc3,$acc3,$acc3
607	adde $acc4,$acc4,$acc4
608	adde $acc5,$acc5,$acc5
609	adde $acc6,$acc6,$acc6
610	li $acc7,0
611	addze $acc7,$acc7
612
613	mulld $acc0,$a0,$a0 # a[0]*a[0]
614	mulhdu $a0,$a0,$a0
615	mulld $t1,$a1,$a1 # a[1]*a[1]
616	mulhdu $a1,$a1,$a1
617	mulld $t2,$a2,$a2 # a[2]*a[2]
618	mulhdu $a2,$a2,$a2
619	mulld $t3,$a3,$a3 # a[3]*a[3]
620	mulhdu $a3,$a3,$a3
621	addc $acc1,$acc1,$a0 # +a[i]*a[i]
622	sldi $t0,$acc0,32
623	adde $acc2,$acc2,$t1
624	srdi $t1,$acc0,32
625	adde $acc3,$acc3,$a1
626	adde $acc4,$acc4,$t2
627	adde $acc5,$acc5,$a2
628	adde $acc6,$acc6,$t3
629	adde $acc7,$acc7,$a3
630	___
631	for($i=0;$i<3;$i++) { # reductions, see commentary in
632	# multiplication for details
633	$code.=<<___;
634	subfc $t2,$t0,$acc0 # "*0xffff0001"
635	subfe $t3,$t1,$acc0
636	addc $acc0,$acc1,$t0 # +=acc[0]<<96 and omit acc[0]
637	sldi $t0,$acc0,32
638	adde $acc1,$acc2,$t1
639	srdi $t1,$acc0,32
640	adde $acc2,$acc3,$t2 # +=acc[0]*0xffff0001
641	addze $acc3,$t3 # can't overflow
642	___
643	}
644	$code.=<<___;
645	subfc $t2,$t0,$acc0 # "*0xffff0001"
646	subfe $t3,$t1,$acc0
647	addc $acc0,$acc1,$t0 # +=acc[0]<<96 and omit acc[0]
648	adde $acc1,$acc2,$t1
649	adde $acc2,$acc3,$t2 # +=acc[0]*0xffff0001
650	addze $acc3,$t3 # can't overflow
651
652	addc $acc0,$acc0,$acc4 # accumulate upper half
653	adde $acc1,$acc1,$acc5
654	adde $acc2,$acc2,$acc6
655	adde $acc3,$acc3,$acc7
656	li $t2,0
657	addze $acc4,$t2
658
659	addic $acc0,$acc0,1 # ret -= modulus
660	subfe $acc1,$poly1,$acc1
661	subfe $acc2,$t2,$acc2
662	subfe $acc3,$poly3,$acc3
663	subfe $acc4,$t2,$acc4
664
665	addc $acc0,$acc0,$acc4 # ret += modulus if borrow
666	and $t1,$poly1,$acc4
667	and $t3,$poly3,$acc4
668	adde $acc1,$acc1,$t1
669	addze $acc2,$acc2
670	adde $acc3,$acc3,$t3
671
672	std $acc0,0($rp)
673	std $acc1,8($rp)
674	std $acc2,16($rp)
675	std $acc3,24($rp)
676
677	blr
678	.long 0
679	.byte 0,12,0x14,0,0,0,1,0
680	.long 0
681	.size __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont
682
683	# Note that __ecp_nistz256_add expects both input vectors pre-loaded to
684	# $a0-$a3 and $t0-$t3. This is done because it's used in multiple
685	# contexts, e.g. in multiplication by 2 and 3...
686	.type __ecp_nistz256_add,\@function
687	.align 4
688	__ecp_nistz256_add:
689	addc $acc0,$acc0,$t0 # ret = a+b
690	adde $acc1,$acc1,$t1
691	adde $acc2,$acc2,$t2
692	li $t2,0
693	adde $acc3,$acc3,$t3
694	addze $t0,$t2
695
696	# if a+b >= modulus, subtract modulus
697	#
698	# But since comparison implies subtraction, we subtract
699	# modulus and then add it back if subtraction borrowed.
700
701	subic $acc0,$acc0,-1
702	subfe $acc1,$poly1,$acc1
703	subfe $acc2,$t2,$acc2
704	subfe $acc3,$poly3,$acc3
705	subfe $t0,$t2,$t0
706
707	addc $acc0,$acc0,$t0
708	and $t1,$poly1,$t0
709	and $t3,$poly3,$t0
710	adde $acc1,$acc1,$t1
711	addze $acc2,$acc2
712	adde $acc3,$acc3,$t3
713
714	std $acc0,0($rp)
715	std $acc1,8($rp)
716	std $acc2,16($rp)
717	std $acc3,24($rp)
718
719	blr
720	.long 0
721	.byte 0,12,0x14,0,0,0,3,0
722	.long 0
723	.size __ecp_nistz256_add,.-__ecp_nistz256_add
724
725	.type __ecp_nistz256_sub_from,\@function
726	.align 4
727	__ecp_nistz256_sub_from:
728	ld $t0,0($bp)
729	ld $t1,8($bp)
730	ld $t2,16($bp)
731	ld $t3,24($bp)
732	subfc $acc0,$t0,$acc0 # ret = a-b
733	subfe $acc1,$t1,$acc1
734	subfe $acc2,$t2,$acc2
735	subfe $acc3,$t3,$acc3
736	subfe $t0,$t0,$t0 # t0 = borrow ? -1 : 0
737
738	# if a-b borrowed, add modulus
739
740	addc $acc0,$acc0,$t0 # ret -= modulus & t0
741	and $t1,$poly1,$t0
742	and $t3,$poly3,$t0
743	adde $acc1,$acc1,$t1
744	addze $acc2,$acc2
745	adde $acc3,$acc3,$t3
746
747	std $acc0,0($rp)
748	std $acc1,8($rp)
749	std $acc2,16($rp)
750	std $acc3,24($rp)
751
752	blr
753	.long 0
754	.byte 0,12,0x14,0,0,0,3,0
755	.long 0
756	.size __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from
757
758	.type __ecp_nistz256_sub_morf,\@function
759	.align 4
760	__ecp_nistz256_sub_morf:
761	ld $t0,0($bp)
762	ld $t1,8($bp)
763	ld $t2,16($bp)
764	ld $t3,24($bp)
765	subfc $acc0,$acc0,$t0 # ret = b-a
766	subfe $acc1,$acc1,$t1
767	subfe $acc2,$acc2,$t2
768	subfe $acc3,$acc3,$t3
769	subfe $t0,$t0,$t0 # t0 = borrow ? -1 : 0
770
771	# if b-a borrowed, add modulus
772
773	addc $acc0,$acc0,$t0 # ret -= modulus & t0
774	and $t1,$poly1,$t0
775	and $t3,$poly3,$t0
776	adde $acc1,$acc1,$t1
777	addze $acc2,$acc2
778	adde $acc3,$acc3,$t3
779
780	std $acc0,0($rp)
781	std $acc1,8($rp)
782	std $acc2,16($rp)
783	std $acc3,24($rp)
784
785	blr
786	.long 0
787	.byte 0,12,0x14,0,0,0,3,0
788	.long 0
789	.size __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf
790
791	.type __ecp_nistz256_div_by_2,\@function
792	.align 4
793	__ecp_nistz256_div_by_2:
794	andi. $t0,$acc0,1
795	addic $acc0,$acc0,-1 # a += modulus
796	neg $t0,$t0
797	adde $acc1,$acc1,$poly1
798	not $t0,$t0
799	addze $acc2,$acc2
800	li $t2,0
801	adde $acc3,$acc3,$poly3
802	and $t1,$poly1,$t0
803	addze $ap,$t2 # ap = carry
804	and $t3,$poly3,$t0
805
806	subfc $acc0,$t0,$acc0 # a -= modulus if a was even
807	subfe $acc1,$t1,$acc1
808	subfe $acc2,$t2,$acc2
809	subfe $acc3,$t3,$acc3
810	subfe $ap, $t2,$ap
811
812	srdi $acc0,$acc0,1
813	sldi $t0,$acc1,63
814	srdi $acc1,$acc1,1
815	sldi $t1,$acc2,63
816	srdi $acc2,$acc2,1
817	sldi $t2,$acc3,63
818	srdi $acc3,$acc3,1
819	sldi $t3,$ap,63
820	or $acc0,$acc0,$t0
821	or $acc1,$acc1,$t1
822	or $acc2,$acc2,$t2
823	or $acc3,$acc3,$t3
824
825	std $acc0,0($rp)
826	std $acc1,8($rp)
827	std $acc2,16($rp)
828	std $acc3,24($rp)
829
830	blr
831	.long 0
832	.byte 0,12,0x14,0,0,0,1,0
833	.long 0
834	.size __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2
835	___
836	########################################################################
837	# following subroutines are "literal" implementation of those found in
838	# ecp_nistz256.c
839	#
840	########################################################################
841	# void ecp_nistz256_point_double(P256_POINT out,const P256_POINT inp);
842	#
843	if (1) {
844	my $FRAME=64+324+128;
845	my ($S,$M,$Zsqr,$tmp0)=map(64+32*$_,(0..3));
846	# above map() describes stack layout with 4 temporary
847	# 256-bit vectors on top.
848	my ($rp_real,$ap_real) = map("r$_",(20,21));
849
850	$code.=<<___;
851	.globl ecp_nistz256_point_double
852	.align 5
853	ecp_nistz256_point_double:
854	stdu $sp,-$FRAME($sp)
855	mflr r0
856	std r20,$FRAME-8*12($sp)
857	std r21,$FRAME-8*11($sp)
858	std r22,$FRAME-8*10($sp)
859	std r23,$FRAME-8*9($sp)
860	std r24,$FRAME-8*8($sp)
861	std r25,$FRAME-8*7($sp)
862	std r26,$FRAME-8*6($sp)
863	std r27,$FRAME-8*5($sp)
864	std r28,$FRAME-8*4($sp)
865	std r29,$FRAME-8*3($sp)
866	std r30,$FRAME-8*2($sp)
867	std r31,$FRAME-8*1($sp)
868
869	li $poly1,-1
870	srdi $poly1,$poly1,32 # 0x00000000ffffffff
871	li $poly3,1
872	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
873	.Ldouble_shortcut:
874	ld $acc0,32($ap)
875	ld $acc1,40($ap)
876	ld $acc2,48($ap)
877	ld $acc3,56($ap)
878	mr $t0,$acc0
879	mr $t1,$acc1
880	mr $t2,$acc2
881	mr $t3,$acc3
882	ld $a0,64($ap) # forward load for p256_sqr_mont
883	ld $a1,72($ap)
884	ld $a2,80($ap)
885	ld $a3,88($ap)
886	mr $rp_real,$rp
887	mr $ap_real,$ap
888	addi $rp,$sp,$S
889	bl __ecp_nistz256_add # p256_mul_by_2(S, in_y);
890
891	addi $rp,$sp,$Zsqr
892	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(Zsqr, in_z);
893
894	ld $t0,0($ap_real)
895	ld $t1,8($ap_real)
896	ld $t2,16($ap_real)
897	ld $t3,24($ap_real)
898	mr $a0,$acc0 # put Zsqr aside for p256_sub
899	mr $a1,$acc1
900	mr $a2,$acc2
901	mr $a3,$acc3
902	addi $rp,$sp,$M
903	bl __ecp_nistz256_add # p256_add(M, Zsqr, in_x);
904
905	addi $bp,$ap_real,0
906	mr $acc0,$a0 # restore Zsqr
907	mr $acc1,$a1
908	mr $acc2,$a2
909	mr $acc3,$a3
910	ld $a0,$S+0($sp) # forward load for p256_sqr_mont
911	ld $a1,$S+8($sp)
912	ld $a2,$S+16($sp)
913	ld $a3,$S+24($sp)
914	addi $rp,$sp,$Zsqr
915	bl __ecp_nistz256_sub_morf # p256_sub(Zsqr, in_x, Zsqr);
916
917	addi $rp,$sp,$S
918	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(S, S);
919
920	ld $bi,32($ap_real)
921	ld $a0,64($ap_real)
922	ld $a1,72($ap_real)
923	ld $a2,80($ap_real)
924	ld $a3,88($ap_real)
925	addi $bp,$ap_real,32
926	addi $rp,$sp,$tmp0
927	bl __ecp_nistz256_mul_mont # p256_mul_mont(tmp0, in_z, in_y);
928
929	mr $t0,$acc0
930	mr $t1,$acc1
931	mr $t2,$acc2
932	mr $t3,$acc3
933	ld $a0,$S+0($sp) # forward load for p256_sqr_mont
934	ld $a1,$S+8($sp)
935	ld $a2,$S+16($sp)
936	ld $a3,$S+24($sp)
937	addi $rp,$rp_real,64
938	bl __ecp_nistz256_add # p256_mul_by_2(res_z, tmp0);
939
940	addi $rp,$sp,$tmp0
941	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(tmp0, S);
942
943	ld $bi,$Zsqr($sp) # forward load for p256_mul_mont
944	ld $a0,$M+0($sp)
945	ld $a1,$M+8($sp)
946	ld $a2,$M+16($sp)
947	ld $a3,$M+24($sp)
948	addi $rp,$rp_real,32
949	bl __ecp_nistz256_div_by_2 # p256_div_by_2(res_y, tmp0);
950
951	addi $bp,$sp,$Zsqr
952	addi $rp,$sp,$M
953	bl __ecp_nistz256_mul_mont # p256_mul_mont(M, M, Zsqr);
954
955	mr $t0,$acc0 # duplicate M
956	mr $t1,$acc1
957	mr $t2,$acc2
958	mr $t3,$acc3
959	mr $a0,$acc0 # put M aside
960	mr $a1,$acc1
961	mr $a2,$acc2
962	mr $a3,$acc3
963	addi $rp,$sp,$M
964	bl __ecp_nistz256_add
965	mr $t0,$a0 # restore M
966	mr $t1,$a1
967	mr $t2,$a2
968	mr $t3,$a3
969	ld $bi,0($ap_real) # forward load for p256_mul_mont
970	ld $a0,$S+0($sp)
971	ld $a1,$S+8($sp)
972	ld $a2,$S+16($sp)
973	ld $a3,$S+24($sp)
974	bl __ecp_nistz256_add # p256_mul_by_3(M, M);
975
976	addi $bp,$ap_real,0
977	addi $rp,$sp,$S
978	bl __ecp_nistz256_mul_mont # p256_mul_mont(S, S, in_x);
979
980	mr $t0,$acc0
981	mr $t1,$acc1
982	mr $t2,$acc2
983	mr $t3,$acc3
984	ld $a0,$M+0($sp) # forward load for p256_sqr_mont
985	ld $a1,$M+8($sp)
986	ld $a2,$M+16($sp)
987	ld $a3,$M+24($sp)
988	addi $rp,$sp,$tmp0
989	bl __ecp_nistz256_add # p256_mul_by_2(tmp0, S);
990
991	addi $rp,$rp_real,0
992	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(res_x, M);
993
994	addi $bp,$sp,$tmp0
995	bl __ecp_nistz256_sub_from # p256_sub(res_x, res_x, tmp0);
996
997	addi $bp,$sp,$S
998	addi $rp,$sp,$S
999	bl __ecp_nistz256_sub_morf # p256_sub(S, S, res_x);
1000
1001	ld $bi,$M($sp)
1002	mr $a0,$acc0 # copy S
1003	mr $a1,$acc1
1004	mr $a2,$acc2
1005	mr $a3,$acc3
1006	addi $bp,$sp,$M
1007	bl __ecp_nistz256_mul_mont # p256_mul_mont(S, S, M);
1008
1009	addi $bp,$rp_real,32
1010	addi $rp,$rp_real,32
1011	bl __ecp_nistz256_sub_from # p256_sub(res_y, S, res_y);
1012
1013	mtlr r0
1014	ld r20,$FRAME-8*12($sp)
1015	ld r21,$FRAME-8*11($sp)
1016	ld r22,$FRAME-8*10($sp)
1017	ld r23,$FRAME-8*9($sp)
1018	ld r24,$FRAME-8*8($sp)
1019	ld r25,$FRAME-8*7($sp)
1020	ld r26,$FRAME-8*6($sp)
1021	ld r27,$FRAME-8*5($sp)
1022	ld r28,$FRAME-8*4($sp)
1023	ld r29,$FRAME-8*3($sp)
1024	ld r30,$FRAME-8*2($sp)
1025	ld r31,$FRAME-8*1($sp)
1026	addi $sp,$sp,$FRAME
1027	blr
1028	.long 0
1029	.byte 0,12,4,0,0x80,12,2,0
1030	.long 0
1031	.size ecp_nistz256_point_double,.-ecp_nistz256_point_double
1032	___
1033	}
1034
1035	########################################################################
1036	# void ecp_nistz256_point_add(P256_POINT out,const P256_POINT in1,
1037	# const P256_POINT *in2);
1038	if (1) {
1039	my $FRAME = 64 + 3212 + 168;
1040	my ($res_x,$res_y,$res_z,
1041	$H,$Hsqr,$R,$Rsqr,$Hcub,
1042	$U1,$U2,$S1,$S2)=map(64+32*$_,(0..11));
1043	my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
1044	# above map() describes stack layout with 12 temporary
1045	# 256-bit vectors on top.
1046	my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("r$_",(16..21));
1047
1048	$code.=<<___;
1049	.globl ecp_nistz256_point_add
1050	.align 5
1051	ecp_nistz256_point_add:
1052	stdu $sp,-$FRAME($sp)
1053	mflr r0
1054	std r16,$FRAME-8*16($sp)
1055	std r17,$FRAME-8*15($sp)
1056	std r18,$FRAME-8*14($sp)
1057	std r19,$FRAME-8*13($sp)
1058	std r20,$FRAME-8*12($sp)
1059	std r21,$FRAME-8*11($sp)
1060	std r22,$FRAME-8*10($sp)
1061	std r23,$FRAME-8*9($sp)
1062	std r24,$FRAME-8*8($sp)
1063	std r25,$FRAME-8*7($sp)
1064	std r26,$FRAME-8*6($sp)
1065	std r27,$FRAME-8*5($sp)
1066	std r28,$FRAME-8*4($sp)
1067	std r29,$FRAME-8*3($sp)
1068	std r30,$FRAME-8*2($sp)
1069	std r31,$FRAME-8*1($sp)
1070
1071	li $poly1,-1
1072	srdi $poly1,$poly1,32 # 0x00000000ffffffff
1073	li $poly3,1
1074	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
1075
1076	ld $a0,64($bp) # in2_z
1077	ld $a1,72($bp)
1078	ld $a2,80($bp)
1079	ld $a3,88($bp)
1080	mr $rp_real,$rp
1081	mr $ap_real,$ap
1082	mr $bp_real,$bp
1083	or $t0,$a0,$a1
1084	or $t2,$a2,$a3
1085	or $in2infty,$t0,$t2
1086	neg $t0,$in2infty
1087	or $in2infty,$in2infty,$t0
1088	sradi $in2infty,$in2infty,63 # !in2infty
1089	addi $rp,$sp,$Z2sqr
1090	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(Z2sqr, in2_z);
1091
1092	ld $a0,64($ap_real) # in1_z
1093	ld $a1,72($ap_real)
1094	ld $a2,80($ap_real)
1095	ld $a3,88($ap_real)
1096	or $t0,$a0,$a1
1097	or $t2,$a2,$a3
1098	or $in1infty,$t0,$t2
1099	neg $t0,$in1infty
1100	or $in1infty,$in1infty,$t0
1101	sradi $in1infty,$in1infty,63 # !in1infty
1102	addi $rp,$sp,$Z1sqr
1103	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(Z1sqr, in1_z);
1104
1105	ld $bi,64($bp_real)
1106	ld $a0,$Z2sqr+0($sp)
1107	ld $a1,$Z2sqr+8($sp)
1108	ld $a2,$Z2sqr+16($sp)
1109	ld $a3,$Z2sqr+24($sp)
1110	addi $bp,$bp_real,64
1111	addi $rp,$sp,$S1
1112	bl __ecp_nistz256_mul_mont # p256_mul_mont(S1, Z2sqr, in2_z);
1113
1114	ld $bi,64($ap_real)
1115	ld $a0,$Z1sqr+0($sp)
1116	ld $a1,$Z1sqr+8($sp)
1117	ld $a2,$Z1sqr+16($sp)
1118	ld $a3,$Z1sqr+24($sp)
1119	addi $bp,$ap_real,64
1120	addi $rp,$sp,$S2
1121	bl __ecp_nistz256_mul_mont # p256_mul_mont(S2, Z1sqr, in1_z);
1122
1123	ld $bi,32($ap_real)
1124	ld $a0,$S1+0($sp)
1125	ld $a1,$S1+8($sp)
1126	ld $a2,$S1+16($sp)
1127	ld $a3,$S1+24($sp)
1128	addi $bp,$ap_real,32
1129	addi $rp,$sp,$S1
1130	bl __ecp_nistz256_mul_mont # p256_mul_mont(S1, S1, in1_y);
1131
1132	ld $bi,32($bp_real)
1133	ld $a0,$S2+0($sp)
1134	ld $a1,$S2+8($sp)
1135	ld $a2,$S2+16($sp)
1136	ld $a3,$S2+24($sp)
1137	addi $bp,$bp_real,32
1138	addi $rp,$sp,$S2
1139	bl __ecp_nistz256_mul_mont # p256_mul_mont(S2, S2, in2_y);
1140
1141	addi $bp,$sp,$S1
1142	ld $bi,$Z2sqr($sp) # forward load for p256_mul_mont
1143	ld $a0,0($ap_real)
1144	ld $a1,8($ap_real)
1145	ld $a2,16($ap_real)
1146	ld $a3,24($ap_real)
1147	addi $rp,$sp,$R
1148	bl __ecp_nistz256_sub_from # p256_sub(R, S2, S1);
1149
1150	or $acc0,$acc0,$acc1 # see if result is zero
1151	or $acc2,$acc2,$acc3
1152	or $temp,$acc0,$acc2
1153
1154	addi $bp,$sp,$Z2sqr
1155	addi $rp,$sp,$U1
1156	bl __ecp_nistz256_mul_mont # p256_mul_mont(U1, in1_x, Z2sqr);
1157
1158	ld $bi,$Z1sqr($sp)
1159	ld $a0,0($bp_real)
1160	ld $a1,8($bp_real)
1161	ld $a2,16($bp_real)
1162	ld $a3,24($bp_real)
1163	addi $bp,$sp,$Z1sqr
1164	addi $rp,$sp,$U2
1165	bl __ecp_nistz256_mul_mont # p256_mul_mont(U2, in2_x, Z1sqr);
1166
1167	addi $bp,$sp,$U1
1168	ld $a0,$R+0($sp) # forward load for p256_sqr_mont
1169	ld $a1,$R+8($sp)
1170	ld $a2,$R+16($sp)
1171	ld $a3,$R+24($sp)
1172	addi $rp,$sp,$H
1173	bl __ecp_nistz256_sub_from # p256_sub(H, U2, U1);
1174
1175	or $acc0,$acc0,$acc1 # see if result is zero
1176	or $acc2,$acc2,$acc3
1177	or. $acc0,$acc0,$acc2
1178	bne .Ladd_proceed # is_equal(U1,U2)?
1179
1180	and. $t0,$in1infty,$in2infty
1181	beq .Ladd_proceed # (in1infty \|\| in2infty)?
1182
1183	cmpldi $temp,0
1184	beq .Ladd_double # is_equal(S1,S2)?
1185
1186	xor $a0,$a0,$a0
1187	std $a0,0($rp_real)
1188	std $a0,8($rp_real)
1189	std $a0,16($rp_real)
1190	std $a0,24($rp_real)
1191	std $a0,32($rp_real)
1192	std $a0,40($rp_real)
1193	std $a0,48($rp_real)
1194	std $a0,56($rp_real)
1195	std $a0,64($rp_real)
1196	std $a0,72($rp_real)
1197	std $a0,80($rp_real)
1198	std $a0,88($rp_real)
1199	b .Ladd_done
1200
1201	.align 4
1202	.Ladd_double:
1203	ld $bp,0($sp) # back-link
1204	mr $ap,$ap_real
1205	mr $rp,$rp_real
1206	ld r16,$FRAME-8*16($sp)
1207	ld r17,$FRAME-8*15($sp)
1208	ld r18,$FRAME-8*14($sp)
1209	ld r19,$FRAME-8*13($sp)
1210	stdu $bp,$FRAME-288($sp) # difference in stack frame sizes
1211	b .Ldouble_shortcut
1212
1213	.align 4
1214	.Ladd_proceed:
1215	addi $rp,$sp,$Rsqr
1216	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(Rsqr, R);
1217
1218	ld $bi,64($ap_real)
1219	ld $a0,$H+0($sp)
1220	ld $a1,$H+8($sp)
1221	ld $a2,$H+16($sp)
1222	ld $a3,$H+24($sp)
1223	addi $bp,$ap_real,64
1224	addi $rp,$sp,$res_z
1225	bl __ecp_nistz256_mul_mont # p256_mul_mont(res_z, H, in1_z);
1226
1227	ld $a0,$H+0($sp)
1228	ld $a1,$H+8($sp)
1229	ld $a2,$H+16($sp)
1230	ld $a3,$H+24($sp)
1231	addi $rp,$sp,$Hsqr
1232	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(Hsqr, H);
1233
1234	ld $bi,64($bp_real)
1235	ld $a0,$res_z+0($sp)
1236	ld $a1,$res_z+8($sp)
1237	ld $a2,$res_z+16($sp)
1238	ld $a3,$res_z+24($sp)
1239	addi $bp,$bp_real,64
1240	addi $rp,$sp,$res_z
1241	bl __ecp_nistz256_mul_mont # p256_mul_mont(res_z, res_z, in2_z);
1242
1243	ld $bi,$H($sp)
1244	ld $a0,$Hsqr+0($sp)
1245	ld $a1,$Hsqr+8($sp)
1246	ld $a2,$Hsqr+16($sp)
1247	ld $a3,$Hsqr+24($sp)
1248	addi $bp,$sp,$H
1249	addi $rp,$sp,$Hcub
1250	bl __ecp_nistz256_mul_mont # p256_mul_mont(Hcub, Hsqr, H);
1251
1252	ld $bi,$Hsqr($sp)
1253	ld $a0,$U1+0($sp)
1254	ld $a1,$U1+8($sp)
1255	ld $a2,$U1+16($sp)
1256	ld $a3,$U1+24($sp)
1257	addi $bp,$sp,$Hsqr
1258	addi $rp,$sp,$U2
1259	bl __ecp_nistz256_mul_mont # p256_mul_mont(U2, U1, Hsqr);
1260
1261	mr $t0,$acc0
1262	mr $t1,$acc1
1263	mr $t2,$acc2
1264	mr $t3,$acc3
1265	addi $rp,$sp,$Hsqr
1266	bl __ecp_nistz256_add # p256_mul_by_2(Hsqr, U2);
1267
1268	addi $bp,$sp,$Rsqr
1269	addi $rp,$sp,$res_x
1270	bl __ecp_nistz256_sub_morf # p256_sub(res_x, Rsqr, Hsqr);
1271
1272	addi $bp,$sp,$Hcub
1273	bl __ecp_nistz256_sub_from # p256_sub(res_x, res_x, Hcub);
1274
1275	addi $bp,$sp,$U2
1276	ld $bi,$Hcub($sp) # forward load for p256_mul_mont
1277	ld $a0,$S1+0($sp)
1278	ld $a1,$S1+8($sp)
1279	ld $a2,$S1+16($sp)
1280	ld $a3,$S1+24($sp)
1281	addi $rp,$sp,$res_y
1282	bl __ecp_nistz256_sub_morf # p256_sub(res_y, U2, res_x);
1283
1284	addi $bp,$sp,$Hcub
1285	addi $rp,$sp,$S2
1286	bl __ecp_nistz256_mul_mont # p256_mul_mont(S2, S1, Hcub);
1287
1288	ld $bi,$R($sp)
1289	ld $a0,$res_y+0($sp)
1290	ld $a1,$res_y+8($sp)
1291	ld $a2,$res_y+16($sp)
1292	ld $a3,$res_y+24($sp)
1293	addi $bp,$sp,$R
1294	addi $rp,$sp,$res_y
1295	bl __ecp_nistz256_mul_mont # p256_mul_mont(res_y, res_y, R);
1296
1297	addi $bp,$sp,$S2
1298	bl __ecp_nistz256_sub_from # p256_sub(res_y, res_y, S2);
1299
1300	ld $t0,0($bp_real) # in2
1301	ld $t1,8($bp_real)
1302	ld $t2,16($bp_real)
1303	ld $t3,24($bp_real)
1304	ld $a0,$res_x+0($sp) # res
1305	ld $a1,$res_x+8($sp)
1306	ld $a2,$res_x+16($sp)
1307	ld $a3,$res_x+24($sp)
1308	___
1309	for($i=0;$i<64;$i+=32) { # conditional moves
1310	$code.=<<___;
1311	ld $acc0,$i+0($ap_real) # in1
1312	ld $acc1,$i+8($ap_real)
1313	ld $acc2,$i+16($ap_real)
1314	ld $acc3,$i+24($ap_real)
1315	andc $t0,$t0,$in1infty
1316	andc $t1,$t1,$in1infty
1317	andc $t2,$t2,$in1infty
1318	andc $t3,$t3,$in1infty
1319	and $a0,$a0,$in1infty
1320	and $a1,$a1,$in1infty
1321	and $a2,$a2,$in1infty
1322	and $a3,$a3,$in1infty
1323	or $t0,$t0,$a0
1324	or $t1,$t1,$a1
1325	or $t2,$t2,$a2
1326	or $t3,$t3,$a3
1327	andc $acc0,$acc0,$in2infty
1328	andc $acc1,$acc1,$in2infty
1329	andc $acc2,$acc2,$in2infty
1330	andc $acc3,$acc3,$in2infty
1331	and $t0,$t0,$in2infty
1332	and $t1,$t1,$in2infty
1333	and $t2,$t2,$in2infty
1334	and $t3,$t3,$in2infty
1335	or $acc0,$acc0,$t0
1336	or $acc1,$acc1,$t1
1337	or $acc2,$acc2,$t2
1338	or $acc3,$acc3,$t3
1339
1340	ld $t0,$i+32($bp_real) # in2
1341	ld $t1,$i+40($bp_real)
1342	ld $t2,$i+48($bp_real)
1343	ld $t3,$i+56($bp_real)
1344	ld $a0,$res_x+$i+32($sp)
1345	ld $a1,$res_x+$i+40($sp)
1346	ld $a2,$res_x+$i+48($sp)
1347	ld $a3,$res_x+$i+56($sp)
1348	std $acc0,$i+0($rp_real)
1349	std $acc1,$i+8($rp_real)
1350	std $acc2,$i+16($rp_real)
1351	std $acc3,$i+24($rp_real)
1352	___
1353	}
1354	$code.=<<___;
1355	ld $acc0,$i+0($ap_real) # in1
1356	ld $acc1,$i+8($ap_real)
1357	ld $acc2,$i+16($ap_real)
1358	ld $acc3,$i+24($ap_real)
1359	andc $t0,$t0,$in1infty
1360	andc $t1,$t1,$in1infty
1361	andc $t2,$t2,$in1infty
1362	andc $t3,$t3,$in1infty
1363	and $a0,$a0,$in1infty
1364	and $a1,$a1,$in1infty
1365	and $a2,$a2,$in1infty
1366	and $a3,$a3,$in1infty
1367	or $t0,$t0,$a0
1368	or $t1,$t1,$a1
1369	or $t2,$t2,$a2
1370	or $t3,$t3,$a3
1371	andc $acc0,$acc0,$in2infty
1372	andc $acc1,$acc1,$in2infty
1373	andc $acc2,$acc2,$in2infty
1374	andc $acc3,$acc3,$in2infty
1375	and $t0,$t0,$in2infty
1376	and $t1,$t1,$in2infty
1377	and $t2,$t2,$in2infty
1378	and $t3,$t3,$in2infty
1379	or $acc0,$acc0,$t0
1380	or $acc1,$acc1,$t1
1381	or $acc2,$acc2,$t2
1382	or $acc3,$acc3,$t3
1383	std $acc0,$i+0($rp_real)
1384	std $acc1,$i+8($rp_real)
1385	std $acc2,$i+16($rp_real)
1386	std $acc3,$i+24($rp_real)
1387
1388	.Ladd_done:
1389	mtlr r0
1390	ld r16,$FRAME-8*16($sp)
1391	ld r17,$FRAME-8*15($sp)
1392	ld r18,$FRAME-8*14($sp)
1393	ld r19,$FRAME-8*13($sp)
1394	ld r20,$FRAME-8*12($sp)
1395	ld r21,$FRAME-8*11($sp)
1396	ld r22,$FRAME-8*10($sp)
1397	ld r23,$FRAME-8*9($sp)
1398	ld r24,$FRAME-8*8($sp)
1399	ld r25,$FRAME-8*7($sp)
1400	ld r26,$FRAME-8*6($sp)
1401	ld r27,$FRAME-8*5($sp)
1402	ld r28,$FRAME-8*4($sp)
1403	ld r29,$FRAME-8*3($sp)
1404	ld r30,$FRAME-8*2($sp)
1405	ld r31,$FRAME-8*1($sp)
1406	addi $sp,$sp,$FRAME
1407	blr
1408	.long 0
1409	.byte 0,12,4,0,0x80,16,3,0
1410	.long 0
1411	.size ecp_nistz256_point_add,.-ecp_nistz256_point_add
1412	___
1413	}
1414
1415	########################################################################
1416	# void ecp_nistz256_point_add_affine(P256_POINT out,const P256_POINT in1,
1417	# const P256_POINT_AFFINE *in2);
1418	if (1) {
1419	my $FRAME = 64 + 3210 + 168;
1420	my ($res_x,$res_y,$res_z,
1421	$U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(64+32*$_,(0..9));
1422	my $Z1sqr = $S2;
1423	# above map() describes stack layout with 10 temporary
1424	# 256-bit vectors on top.
1425	my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("r$_",(16..21));
1426
1427	$code.=<<___;
1428	.globl ecp_nistz256_point_add_affine
1429	.align 5
1430	ecp_nistz256_point_add_affine:
1431	stdu $sp,-$FRAME($sp)
1432	mflr r0
1433	std r16,$FRAME-8*16($sp)
1434	std r17,$FRAME-8*15($sp)
1435	std r18,$FRAME-8*14($sp)
1436	std r19,$FRAME-8*13($sp)
1437	std r20,$FRAME-8*12($sp)
1438	std r21,$FRAME-8*11($sp)
1439	std r22,$FRAME-8*10($sp)
1440	std r23,$FRAME-8*9($sp)
1441	std r24,$FRAME-8*8($sp)
1442	std r25,$FRAME-8*7($sp)
1443	std r26,$FRAME-8*6($sp)
1444	std r27,$FRAME-8*5($sp)
1445	std r28,$FRAME-8*4($sp)
1446	std r29,$FRAME-8*3($sp)
1447	std r30,$FRAME-8*2($sp)
1448	std r31,$FRAME-8*1($sp)
1449
1450	li $poly1,-1
1451	srdi $poly1,$poly1,32 # 0x00000000ffffffff
1452	li $poly3,1
1453	orc $poly3,$poly3,$poly1 # 0xffffffff00000001
1454
1455	mr $rp_real,$rp
1456	mr $ap_real,$ap
1457	mr $bp_real,$bp
1458
1459	ld $a0,64($ap) # in1_z
1460	ld $a1,72($ap)
1461	ld $a2,80($ap)
1462	ld $a3,88($ap)
1463	or $t0,$a0,$a1
1464	or $t2,$a2,$a3
1465	or $in1infty,$t0,$t2
1466	neg $t0,$in1infty
1467	or $in1infty,$in1infty,$t0
1468	sradi $in1infty,$in1infty,63 # !in1infty
1469
1470	ld $acc0,0($bp) # in2_x
1471	ld $acc1,8($bp)
1472	ld $acc2,16($bp)
1473	ld $acc3,24($bp)
1474	ld $t0,32($bp) # in2_y
1475	ld $t1,40($bp)
1476	ld $t2,48($bp)
1477	ld $t3,56($bp)
1478	or $acc0,$acc0,$acc1
1479	or $acc2,$acc2,$acc3
1480	or $acc0,$acc0,$acc2
1481	or $t0,$t0,$t1
1482	or $t2,$t2,$t3
1483	or $t0,$t0,$t2
1484	or $in2infty,$acc0,$t0
1485	neg $t0,$in2infty
1486	or $in2infty,$in2infty,$t0
1487	sradi $in2infty,$in2infty,63 # !in2infty
1488
1489	addi $rp,$sp,$Z1sqr
1490	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(Z1sqr, in1_z);
1491
1492	mr $a0,$acc0
1493	mr $a1,$acc1
1494	mr $a2,$acc2
1495	mr $a3,$acc3
1496	ld $bi,0($bp_real)
1497	addi $bp,$bp_real,0
1498	addi $rp,$sp,$U2
1499	bl __ecp_nistz256_mul_mont # p256_mul_mont(U2, Z1sqr, in2_x);
1500
1501	addi $bp,$ap_real,0
1502	ld $bi,64($ap_real) # forward load for p256_mul_mont
1503	ld $a0,$Z1sqr+0($sp)
1504	ld $a1,$Z1sqr+8($sp)
1505	ld $a2,$Z1sqr+16($sp)
1506	ld $a3,$Z1sqr+24($sp)
1507	addi $rp,$sp,$H
1508	bl __ecp_nistz256_sub_from # p256_sub(H, U2, in1_x);
1509
1510	addi $bp,$ap_real,64
1511	addi $rp,$sp,$S2
1512	bl __ecp_nistz256_mul_mont # p256_mul_mont(S2, Z1sqr, in1_z);
1513
1514	ld $bi,64($ap_real)
1515	ld $a0,$H+0($sp)
1516	ld $a1,$H+8($sp)
1517	ld $a2,$H+16($sp)
1518	ld $a3,$H+24($sp)
1519	addi $bp,$ap_real,64
1520	addi $rp,$sp,$res_z
1521	bl __ecp_nistz256_mul_mont # p256_mul_mont(res_z, H, in1_z);
1522
1523	ld $bi,32($bp_real)
1524	ld $a0,$S2+0($sp)
1525	ld $a1,$S2+8($sp)
1526	ld $a2,$S2+16($sp)
1527	ld $a3,$S2+24($sp)
1528	addi $bp,$bp_real,32
1529	addi $rp,$sp,$S2
1530	bl __ecp_nistz256_mul_mont # p256_mul_mont(S2, S2, in2_y);
1531
1532	addi $bp,$ap_real,32
1533	ld $a0,$H+0($sp) # forward load for p256_sqr_mont
1534	ld $a1,$H+8($sp)
1535	ld $a2,$H+16($sp)
1536	ld $a3,$H+24($sp)
1537	addi $rp,$sp,$R
1538	bl __ecp_nistz256_sub_from # p256_sub(R, S2, in1_y);
1539
1540	addi $rp,$sp,$Hsqr
1541	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(Hsqr, H);
1542
1543	ld $a0,$R+0($sp)
1544	ld $a1,$R+8($sp)
1545	ld $a2,$R+16($sp)
1546	ld $a3,$R+24($sp)
1547	addi $rp,$sp,$Rsqr
1548	bl __ecp_nistz256_sqr_mont # p256_sqr_mont(Rsqr, R);
1549
1550	ld $bi,$H($sp)
1551	ld $a0,$Hsqr+0($sp)
1552	ld $a1,$Hsqr+8($sp)
1553	ld $a2,$Hsqr+16($sp)
1554	ld $a3,$Hsqr+24($sp)
1555	addi $bp,$sp,$H
1556	addi $rp,$sp,$Hcub
1557	bl __ecp_nistz256_mul_mont # p256_mul_mont(Hcub, Hsqr, H);
1558
1559	ld $bi,0($ap_real)
1560	ld $a0,$Hsqr+0($sp)
1561	ld $a1,$Hsqr+8($sp)
1562	ld $a2,$Hsqr+16($sp)
1563	ld $a3,$Hsqr+24($sp)
1564	addi $bp,$ap_real,0
1565	addi $rp,$sp,$U2
1566	bl __ecp_nistz256_mul_mont # p256_mul_mont(U2, in1_x, Hsqr);
1567
1568	mr $t0,$acc0
1569	mr $t1,$acc1
1570	mr $t2,$acc2
1571	mr $t3,$acc3
1572	addi $rp,$sp,$Hsqr
1573	bl __ecp_nistz256_add # p256_mul_by_2(Hsqr, U2);
1574
1575	addi $bp,$sp,$Rsqr
1576	addi $rp,$sp,$res_x
1577	bl __ecp_nistz256_sub_morf # p256_sub(res_x, Rsqr, Hsqr);
1578
1579	addi $bp,$sp,$Hcub
1580	bl __ecp_nistz256_sub_from # p256_sub(res_x, res_x, Hcub);
1581
1582	addi $bp,$sp,$U2
1583	ld $bi,32($ap_real) # forward load for p256_mul_mont
1584	ld $a0,$Hcub+0($sp)
1585	ld $a1,$Hcub+8($sp)
1586	ld $a2,$Hcub+16($sp)
1587	ld $a3,$Hcub+24($sp)
1588	addi $rp,$sp,$res_y
1589	bl __ecp_nistz256_sub_morf # p256_sub(res_y, U2, res_x);
1590
1591	addi $bp,$ap_real,32
1592	addi $rp,$sp,$S2
1593	bl __ecp_nistz256_mul_mont # p256_mul_mont(S2, in1_y, Hcub);
1594
1595	ld $bi,$R($sp)
1596	ld $a0,$res_y+0($sp)
1597	ld $a1,$res_y+8($sp)
1598	ld $a2,$res_y+16($sp)
1599	ld $a3,$res_y+24($sp)
1600	addi $bp,$sp,$R
1601	addi $rp,$sp,$res_y
1602	bl __ecp_nistz256_mul_mont # p256_mul_mont(res_y, res_y, R);
1603
1604	addi $bp,$sp,$S2
1605	bl __ecp_nistz256_sub_from # p256_sub(res_y, res_y, S2);
1606
1607	ld $t0,0($bp_real) # in2
1608	ld $t1,8($bp_real)
1609	ld $t2,16($bp_real)
1610	ld $t3,24($bp_real)
1611	ld $a0,$res_x+0($sp) # res
1612	ld $a1,$res_x+8($sp)
1613	ld $a2,$res_x+16($sp)
1614	ld $a3,$res_x+24($sp)
1615	___
1616	for($i=0;$i<64;$i+=32) { # conditional moves
1617	$code.=<<___;
1618	ld $acc0,$i+0($ap_real) # in1
1619	ld $acc1,$i+8($ap_real)
1620	ld $acc2,$i+16($ap_real)
1621	ld $acc3,$i+24($ap_real)
1622	andc $t0,$t0,$in1infty
1623	andc $t1,$t1,$in1infty
1624	andc $t2,$t2,$in1infty
1625	andc $t3,$t3,$in1infty
1626	and $a0,$a0,$in1infty
1627	and $a1,$a1,$in1infty
1628	and $a2,$a2,$in1infty
1629	and $a3,$a3,$in1infty
1630	or $t0,$t0,$a0
1631	or $t1,$t1,$a1
1632	or $t2,$t2,$a2
1633	or $t3,$t3,$a3
1634	andc $acc0,$acc0,$in2infty
1635	andc $acc1,$acc1,$in2infty
1636	andc $acc2,$acc2,$in2infty
1637	andc $acc3,$acc3,$in2infty
1638	and $t0,$t0,$in2infty
1639	and $t1,$t1,$in2infty
1640	and $t2,$t2,$in2infty
1641	and $t3,$t3,$in2infty
1642	or $acc0,$acc0,$t0
1643	or $acc1,$acc1,$t1
1644	or $acc2,$acc2,$t2
1645	or $acc3,$acc3,$t3
1646	___
1647	$code.=<<___ if ($i==0);
1648	ld $t0,32($bp_real) # in2
1649	ld $t1,40($bp_real)
1650	ld $t2,48($bp_real)
1651	ld $t3,56($bp_real)
1652	___
1653	$code.=<<___ if ($i==32);
1654	li $t0,1 # Lone_mont
1655	not $t1,$poly1
1656	li $t2,-1
1657	not $t3,$poly3
1658	___
1659	$code.=<<___;
1660	ld $a0,$res_x+$i+32($sp)
1661	ld $a1,$res_x+$i+40($sp)
1662	ld $a2,$res_x+$i+48($sp)
1663	ld $a3,$res_x+$i+56($sp)
1664	std $acc0,$i+0($rp_real)
1665	std $acc1,$i+8($rp_real)
1666	std $acc2,$i+16($rp_real)
1667	std $acc3,$i+24($rp_real)
1668	___
1669	}
1670	$code.=<<___;
1671	ld $acc0,$i+0($ap_real) # in1
1672	ld $acc1,$i+8($ap_real)
1673	ld $acc2,$i+16($ap_real)
1674	ld $acc3,$i+24($ap_real)
1675	andc $t0,$t0,$in1infty
1676	andc $t1,$t1,$in1infty
1677	andc $t2,$t2,$in1infty
1678	andc $t3,$t3,$in1infty
1679	and $a0,$a0,$in1infty
1680	and $a1,$a1,$in1infty
1681	and $a2,$a2,$in1infty
1682	and $a3,$a3,$in1infty
1683	or $t0,$t0,$a0
1684	or $t1,$t1,$a1
1685	or $t2,$t2,$a2
1686	or $t3,$t3,$a3
1687	andc $acc0,$acc0,$in2infty
1688	andc $acc1,$acc1,$in2infty
1689	andc $acc2,$acc2,$in2infty
1690	andc $acc3,$acc3,$in2infty
1691	and $t0,$t0,$in2infty
1692	and $t1,$t1,$in2infty
1693	and $t2,$t2,$in2infty
1694	and $t3,$t3,$in2infty
1695	or $acc0,$acc0,$t0
1696	or $acc1,$acc1,$t1
1697	or $acc2,$acc2,$t2
1698	or $acc3,$acc3,$t3
1699	std $acc0,$i+0($rp_real)
1700	std $acc1,$i+8($rp_real)
1701	std $acc2,$i+16($rp_real)
1702	std $acc3,$i+24($rp_real)
1703
1704	mtlr r0
1705	ld r16,$FRAME-8*16($sp)
1706	ld r17,$FRAME-8*15($sp)
1707	ld r18,$FRAME-8*14($sp)
1708	ld r19,$FRAME-8*13($sp)
1709	ld r20,$FRAME-8*12($sp)
1710	ld r21,$FRAME-8*11($sp)
1711	ld r22,$FRAME-8*10($sp)
1712	ld r23,$FRAME-8*9($sp)
1713	ld r24,$FRAME-8*8($sp)
1714	ld r25,$FRAME-8*7($sp)
1715	ld r26,$FRAME-8*6($sp)
1716	ld r27,$FRAME-8*5($sp)
1717	ld r28,$FRAME-8*4($sp)
1718	ld r29,$FRAME-8*3($sp)
1719	ld r30,$FRAME-8*2($sp)
1720	ld r31,$FRAME-8*1($sp)
1721	addi $sp,$sp,$FRAME
1722	blr
1723	.long 0
1724	.byte 0,12,4,0,0x80,16,3,0
1725	.long 0
1726	.size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine
1727	___
1728	}
1729	if (1) {
1730	my ($ordk,$ord0,$ord1,$t4) = map("r$_",(18..21));
1731	my ($ord2,$ord3,$zr) = ($poly1,$poly3,"r0");
1732
1733	$code.=<<___;
1734	########################################################################
1735	# void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4],
1736	# uint64_t b[4]);
1737	.globl ecp_nistz256_ord_mul_mont
1738	.align 5
1739	ecp_nistz256_ord_mul_mont:
1740	stdu $sp,-160($sp)
1741	std r18,48($sp)
1742	std r19,56($sp)
1743	std r20,64($sp)
1744	std r21,72($sp)
1745	std r22,80($sp)
1746	std r23,88($sp)
1747	std r24,96($sp)
1748	std r25,104($sp)
1749	std r26,112($sp)
1750	std r27,120($sp)
1751	std r28,128($sp)
1752	std r29,136($sp)
1753	std r30,144($sp)
1754	std r31,152($sp)
1755
1756	ld $a0,0($ap)
1757	ld $bi,0($bp)
1758	ld $a1,8($ap)
1759	ld $a2,16($ap)
1760	ld $a3,24($ap)
1761
1762	lis $ordk,0xccd1
1763	lis $ord0,0xf3b9
1764	lis $ord1,0xbce6
1765	ori $ordk,$ordk,0xc8aa
1766	ori $ord0,$ord0,0xcac2
1767	ori $ord1,$ord1,0xfaad
1768	sldi $ordk,$ordk,32
1769	sldi $ord0,$ord0,32
1770	sldi $ord1,$ord1,32
1771	oris $ordk,$ordk,0xee00
1772	oris $ord0,$ord0,0xfc63
1773	oris $ord1,$ord1,0xa717
1774	ori $ordk,$ordk,0xbc4f # 0xccd1c8aaee00bc4f
1775	ori $ord0,$ord0,0x2551 # 0xf3b9cac2fc632551
1776	ori $ord1,$ord1,0x9e84 # 0xbce6faada7179e84
1777	li $ord2,-1 # 0xffffffffffffffff
1778	sldi $ord3,$ord2,32 # 0xffffffff00000000
1779	li $zr,0
1780
1781	mulld $acc0,$a0,$bi # a[0]*b[0]
1782	mulhdu $t0,$a0,$bi
1783
1784	mulld $acc1,$a1,$bi # a[1]*b[0]
1785	mulhdu $t1,$a1,$bi
1786
1787	mulld $acc2,$a2,$bi # a[2]*b[0]
1788	mulhdu $t2,$a2,$bi
1789
1790	mulld $acc3,$a3,$bi # a[3]*b[0]
1791	mulhdu $acc4,$a3,$bi
1792
1793	mulld $t4,$acc0,$ordk
1794
1795	addc $acc1,$acc1,$t0 # accumulate high parts of multiplication
1796	adde $acc2,$acc2,$t1
1797	adde $acc3,$acc3,$t2
1798	addze $acc4,$acc4
1799	li $acc5,0
1800	___
1801	for ($i=1;$i<4;$i++) {
1802	################################################################
1803	# ffff0000.ffffffff.yyyyyyyy.zzzzzzzz
1804	# * abcdefgh
1805	# + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
1806	#
1807	# Now observing that ff..ffx = (2^n-1)x = 2^n*x-x, we
1808	# rewrite above as:
1809	#
1810	# xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
1811	# - 0000abcd.efgh0000.abcdefgh.00000000.00000000
1812	# + abcdefgh.abcdefgh.yzayzbyz.cyzdyzey.zfyzgyzh
1813	$code.=<<___;
1814	ld $bi,8*$i($bp) # b[i]
1815
1816	sldi $t0,$t4,32
1817	subfc $acc2,$t4,$acc2
1818	srdi $t1,$t4,32
1819	subfe $acc3,$t0,$acc3
1820	subfe $acc4,$t1,$acc4
1821	subfe $acc5,$zr,$acc5
1822
1823	addic $t0,$acc0,-1 # discarded
1824	mulhdu $t1,$ord0,$t4
1825	mulld $t2,$ord1,$t4
1826	mulhdu $t3,$ord1,$t4
1827
1828	adde $t2,$t2,$t1
1829	mulld $t0,$a0,$bi
1830	addze $t3,$t3
1831	mulld $t1,$a1,$bi
1832
1833	addc $acc0,$acc1,$t2
1834	mulld $t2,$a2,$bi
1835	adde $acc1,$acc2,$t3
1836	mulld $t3,$a3,$bi
1837	adde $acc2,$acc3,$t4
1838	adde $acc3,$acc4,$t4
1839	addze $acc4,$acc5
1840
1841	addc $acc0,$acc0,$t0 # accumulate low parts
1842	mulhdu $t0,$a0,$bi
1843	adde $acc1,$acc1,$t1
1844	mulhdu $t1,$a1,$bi
1845	adde $acc2,$acc2,$t2
1846	mulhdu $t2,$a2,$bi
1847	adde $acc3,$acc3,$t3
1848	mulhdu $t3,$a3,$bi
1849	addze $acc4,$acc4
1850	mulld $t4,$acc0,$ordk
1851	addc $acc1,$acc1,$t0 # accumulate high parts
1852	adde $acc2,$acc2,$t1
1853	adde $acc3,$acc3,$t2
1854	adde $acc4,$acc4,$t3
1855	addze $acc5,$zr
1856	___
1857	}
1858	$code.=<<___;
1859	sldi $t0,$t4,32 # last reduction
1860	subfc $acc2,$t4,$acc2
1861	srdi $t1,$t4,32
1862	subfe $acc3,$t0,$acc3
1863	subfe $acc4,$t1,$acc4
1864	subfe $acc5,$zr,$acc5
1865
1866	addic $t0,$acc0,-1 # discarded
1867	mulhdu $t1,$ord0,$t4
1868	mulld $t2,$ord1,$t4
1869	mulhdu $t3,$ord1,$t4
1870
1871	adde $t2,$t2,$t1
1872	addze $t3,$t3
1873
1874	addc $acc0,$acc1,$t2
1875	adde $acc1,$acc2,$t3
1876	adde $acc2,$acc3,$t4
1877	adde $acc3,$acc4,$t4
1878	addze $acc4,$acc5
1879
1880	subfc $acc0,$ord0,$acc0 # ret -= modulus
1881	subfe $acc1,$ord1,$acc1
1882	subfe $acc2,$ord2,$acc2
1883	subfe $acc3,$ord3,$acc3
1884	subfe $acc4,$zr,$acc4
1885
1886	and $t0,$ord0,$acc4
1887	and $t1,$ord1,$acc4
1888	addc $acc0,$acc0,$t0 # ret += modulus if borrow
1889	and $t3,$ord3,$acc4
1890	adde $acc1,$acc1,$t1
1891	adde $acc2,$acc2,$acc4
1892	adde $acc3,$acc3,$t3
1893
1894	std $acc0,0($rp)
1895	std $acc1,8($rp)
1896	std $acc2,16($rp)
1897	std $acc3,24($rp)
1898
1899	ld r18,48($sp)
1900	ld r19,56($sp)
1901	ld r20,64($sp)
1902	ld r21,72($sp)
1903	ld r22,80($sp)
1904	ld r23,88($sp)
1905	ld r24,96($sp)
1906	ld r25,104($sp)
1907	ld r26,112($sp)
1908	ld r27,120($sp)
1909	ld r28,128($sp)
1910	ld r29,136($sp)
1911	ld r30,144($sp)
1912	ld r31,152($sp)
1913	addi $sp,$sp,160
1914	blr
1915	.long 0
1916	.byte 0,12,4,0,0x80,14,3,0
1917	.long 0
1918	.size ecp_nistz256_ord_mul_mont,.-ecp_nistz256_ord_mul_mont
1919
1920	################################################################################
1921	# void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
1922	# int rep);
1923	.globl ecp_nistz256_ord_sqr_mont
1924	.align 5
1925	ecp_nistz256_ord_sqr_mont:
1926	stdu $sp,-160($sp)
1927	std r18,48($sp)
1928	std r19,56($sp)
1929	std r20,64($sp)
1930	std r21,72($sp)
1931	std r22,80($sp)
1932	std r23,88($sp)
1933	std r24,96($sp)
1934	std r25,104($sp)
1935	std r26,112($sp)
1936	std r27,120($sp)
1937	std r28,128($sp)
1938	std r29,136($sp)
1939	std r30,144($sp)
1940	std r31,152($sp)
1941
1942	mtctr $bp
1943
1944	ld $a0,0($ap)
1945	ld $a1,8($ap)
1946	ld $a2,16($ap)
1947	ld $a3,24($ap)
1948
1949	lis $ordk,0xccd1
1950	lis $ord0,0xf3b9
1951	lis $ord1,0xbce6
1952	ori $ordk,$ordk,0xc8aa
1953	ori $ord0,$ord0,0xcac2
1954	ori $ord1,$ord1,0xfaad
1955	sldi $ordk,$ordk,32
1956	sldi $ord0,$ord0,32
1957	sldi $ord1,$ord1,32
1958	oris $ordk,$ordk,0xee00
1959	oris $ord0,$ord0,0xfc63
1960	oris $ord1,$ord1,0xa717
1961	ori $ordk,$ordk,0xbc4f # 0xccd1c8aaee00bc4f
1962	ori $ord0,$ord0,0x2551 # 0xf3b9cac2fc632551
1963	ori $ord1,$ord1,0x9e84 # 0xbce6faada7179e84
1964	li $ord2,-1 # 0xffffffffffffffff
1965	sldi $ord3,$ord2,32 # 0xffffffff00000000
1966	li $zr,0
1967	b .Loop_ord_sqr
1968
1969	.align 5
1970	.Loop_ord_sqr:
1971	################################################################
1972	# \| \| \| \| \| \|a1*a0\| \|
1973	# \| \| \| \| \|a2*a0\| \| \|
1974	# \| \|a3a2\|a3a0\| \| \| \|
1975	# \| \| \| \|a2*a1\| \| \| \|
1976	# \| \| \|a3*a1\| \| \| \| \|
1977	# *\| \| \| \| \| \| \| \| 2\|
1978	# +\|a3a3\|a2a2\|a1a1\|a0a0\|
1979	# \|--+--+--+--+--+--+--+--\|
1980	# \|A7\|A6\|A5\|A4\|A3\|A2\|A1\|A0\|, where Ax is $accx, i.e. follow $accx
1981	#
1982	# "can't overflow" below mark carrying into high part of
1983	# multiplication result, which can't overflow, because it
1984	# can never be all ones.
1985
1986	mulld $acc1,$a1,$a0 # a[1]*a[0]
1987	mulhdu $t1,$a1,$a0
1988	mulld $acc2,$a2,$a0 # a[2]*a[0]
1989	mulhdu $t2,$a2,$a0
1990	mulld $acc3,$a3,$a0 # a[3]*a[0]
1991	mulhdu $acc4,$a3,$a0
1992
1993	addc $acc2,$acc2,$t1 # accumulate high parts of multiplication
1994	mulld $t0,$a2,$a1 # a[2]*a[1]
1995	mulhdu $t1,$a2,$a1
1996	adde $acc3,$acc3,$t2
1997	mulld $t2,$a3,$a1 # a[3]*a[1]
1998	mulhdu $t3,$a3,$a1
1999	addze $acc4,$acc4 # can't overflow
2000
2001	mulld $acc5,$a3,$a2 # a[3]*a[2]
2002	mulhdu $acc6,$a3,$a2
2003
2004	addc $t1,$t1,$t2 # accumulate high parts of multiplication
2005	mulld $acc0,$a0,$a0 # a[0]*a[0]
2006	addze $t2,$t3 # can't overflow
2007
2008	addc $acc3,$acc3,$t0 # accumulate low parts of multiplication
2009	mulhdu $a0,$a0,$a0
2010	adde $acc4,$acc4,$t1
2011	mulld $t1,$a1,$a1 # a[1]*a[1]
2012	adde $acc5,$acc5,$t2
2013	mulhdu $a1,$a1,$a1
2014	addze $acc6,$acc6 # can't overflow
2015
2016	addc $acc1,$acc1,$acc1 # acc[1-6]*=2
2017	mulld $t2,$a2,$a2 # a[2]*a[2]
2018	adde $acc2,$acc2,$acc2
2019	mulhdu $a2,$a2,$a2
2020	adde $acc3,$acc3,$acc3
2021	mulld $t3,$a3,$a3 # a[3]*a[3]
2022	adde $acc4,$acc4,$acc4
2023	mulhdu $a3,$a3,$a3
2024	adde $acc5,$acc5,$acc5
2025	adde $acc6,$acc6,$acc6
2026	addze $acc7,$zr
2027
2028	addc $acc1,$acc1,$a0 # +a[i]*a[i]
2029	mulld $t4,$acc0,$ordk
2030	adde $acc2,$acc2,$t1
2031	adde $acc3,$acc3,$a1
2032	adde $acc4,$acc4,$t2
2033	adde $acc5,$acc5,$a2
2034	adde $acc6,$acc6,$t3
2035	adde $acc7,$acc7,$a3
2036	___
2037	for($i=0; $i<4; $i++) { # reductions
2038	$code.=<<___;
2039	addic $t0,$acc0,-1 # discarded
2040	mulhdu $t1,$ord0,$t4
2041	mulld $t2,$ord1,$t4
2042	mulhdu $t3,$ord1,$t4
2043
2044	adde $t2,$t2,$t1
2045	addze $t3,$t3
2046
2047	addc $acc0,$acc1,$t2
2048	adde $acc1,$acc2,$t3
2049	adde $acc2,$acc3,$t4
2050	adde $acc3,$zr,$t4 # can't overflow
2051	___
2052	$code.=<<___ if ($i<3);
2053	mulld $t3,$acc0,$ordk
2054	___
2055	$code.=<<___;
2056	sldi $t0,$t4,32
2057	subfc $acc1,$t4,$acc1
2058	srdi $t1,$t4,32
2059	subfe $acc2,$t0,$acc2
2060	subfe $acc3,$t1,$acc3 # can't borrow
2061	___
2062	($t3,$t4) = ($t4,$t3);
2063	}
2064	$code.=<<___;
2065	addc $acc0,$acc0,$acc4 # accumulate upper half
2066	adde $acc1,$acc1,$acc5
2067	adde $acc2,$acc2,$acc6
2068	adde $acc3,$acc3,$acc7
2069	addze $acc4,$zr
2070
2071	subfc $acc0,$ord0,$acc0 # ret -= modulus
2072	subfe $acc1,$ord1,$acc1
2073	subfe $acc2,$ord2,$acc2
2074	subfe $acc3,$ord3,$acc3
2075	subfe $acc4,$zr,$acc4
2076
2077	and $t0,$ord0,$acc4
2078	and $t1,$ord1,$acc4
2079	addc $a0,$acc0,$t0 # ret += modulus if borrow
2080	and $t3,$ord3,$acc4
2081	adde $a1,$acc1,$t1
2082	adde $a2,$acc2,$acc4
2083	adde $a3,$acc3,$t3
2084
2085	bdnz .Loop_ord_sqr
2086
2087	std $a0,0($rp)
2088	std $a1,8($rp)
2089	std $a2,16($rp)
2090	std $a3,24($rp)
2091
2092	ld r18,48($sp)
2093	ld r19,56($sp)
2094	ld r20,64($sp)
2095	ld r21,72($sp)
2096	ld r22,80($sp)
2097	ld r23,88($sp)
2098	ld r24,96($sp)
2099	ld r25,104($sp)
2100	ld r26,112($sp)
2101	ld r27,120($sp)
2102	ld r28,128($sp)
2103	ld r29,136($sp)
2104	ld r30,144($sp)
2105	ld r31,152($sp)
2106	addi $sp,$sp,160
2107	blr
2108	.long 0
2109	.byte 0,12,4,0,0x80,14,3,0
2110	.long 0
2111	.size ecp_nistz256_ord_sqr_mont,.-ecp_nistz256_ord_sqr_mont
2112	___
2113	} }
2114
2115	########################################################################
2116	# scatter-gather subroutines
2117	{
2118	my ($out,$inp,$index,$mask)=map("r$_",(3..7));
2119	$code.=<<___;
2120	########################################################################
2121	# void ecp_nistz256_scatter_w5(void out, const P256_POINT inp,
2122	# int index);
2123	.globl ecp_nistz256_scatter_w5
2124	.align 4
2125	ecp_nistz256_scatter_w5:
2126	slwi $index,$index,2
2127	add $out,$out,$index
2128
2129	ld r8, 0($inp) # X
2130	ld r9, 8($inp)
2131	ld r10,16($inp)
2132	ld r11,24($inp)
2133
2134	stw r8, 64*0-4($out)
2135	srdi r8, r8, 32
2136	stw r9, 64*1-4($out)
2137	srdi r9, r9, 32
2138	stw r10,64*2-4($out)
2139	srdi r10,r10,32
2140	stw r11,64*3-4($out)
2141	srdi r11,r11,32
2142	stw r8, 64*4-4($out)
2143	stw r9, 64*5-4($out)
2144	stw r10,64*6-4($out)
2145	stw r11,64*7-4($out)
2146	addi $out,$out,64*8
2147
2148	ld r8, 32($inp) # Y
2149	ld r9, 40($inp)
2150	ld r10,48($inp)
2151	ld r11,56($inp)
2152
2153	stw r8, 64*0-4($out)
2154	srdi r8, r8, 32
2155	stw r9, 64*1-4($out)
2156	srdi r9, r9, 32
2157	stw r10,64*2-4($out)
2158	srdi r10,r10,32
2159	stw r11,64*3-4($out)
2160	srdi r11,r11,32
2161	stw r8, 64*4-4($out)
2162	stw r9, 64*5-4($out)
2163	stw r10,64*6-4($out)
2164	stw r11,64*7-4($out)
2165	addi $out,$out,64*8
2166
2167	ld r8, 64($inp) # Z
2168	ld r9, 72($inp)
2169	ld r10,80($inp)
2170	ld r11,88($inp)
2171
2172	stw r8, 64*0-4($out)
2173	srdi r8, r8, 32
2174	stw r9, 64*1-4($out)
2175	srdi r9, r9, 32
2176	stw r10,64*2-4($out)
2177	srdi r10,r10,32
2178	stw r11,64*3-4($out)
2179	srdi r11,r11,32
2180	stw r8, 64*4-4($out)
2181	stw r9, 64*5-4($out)
2182	stw r10,64*6-4($out)
2183	stw r11,64*7-4($out)
2184
2185	blr
2186	.long 0
2187	.byte 0,12,0x14,0,0,0,3,0
2188	.long 0
2189	.size ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
2190
2191	########################################################################
2192	# void ecp_nistz256_gather_w5(P256_POINT out, const void inp,
2193	# int index);
2194	.globl ecp_nistz256_gather_w5
2195	.align 4
2196	ecp_nistz256_gather_w5:
2197	neg r0,$index
2198	sradi r0,r0,63
2199
2200	add $index,$index,r0
2201	slwi $index,$index,2
2202	add $inp,$inp,$index
2203
2204	lwz r5, 64*0($inp)
2205	lwz r6, 64*1($inp)
2206	lwz r7, 64*2($inp)
2207	lwz r8, 64*3($inp)
2208	lwz r9, 64*4($inp)
2209	lwz r10,64*5($inp)
2210	lwz r11,64*6($inp)
2211	lwz r12,64*7($inp)
2212	addi $inp,$inp,64*8
2213	sldi r9, r9, 32
2214	sldi r10,r10,32
2215	sldi r11,r11,32
2216	sldi r12,r12,32
2217	or r5,r5,r9
2218	or r6,r6,r10
2219	or r7,r7,r11
2220	or r8,r8,r12
2221	and r5,r5,r0
2222	and r6,r6,r0
2223	and r7,r7,r0
2224	and r8,r8,r0
2225	std r5,0($out) # X
2226	std r6,8($out)
2227	std r7,16($out)
2228	std r8,24($out)
2229
2230	lwz r5, 64*0($inp)
2231	lwz r6, 64*1($inp)
2232	lwz r7, 64*2($inp)
2233	lwz r8, 64*3($inp)
2234	lwz r9, 64*4($inp)
2235	lwz r10,64*5($inp)
2236	lwz r11,64*6($inp)
2237	lwz r12,64*7($inp)
2238	addi $inp,$inp,64*8
2239	sldi r9, r9, 32
2240	sldi r10,r10,32
2241	sldi r11,r11,32
2242	sldi r12,r12,32
2243	or r5,r5,r9
2244	or r6,r6,r10
2245	or r7,r7,r11
2246	or r8,r8,r12
2247	and r5,r5,r0
2248	and r6,r6,r0
2249	and r7,r7,r0
2250	and r8,r8,r0
2251	std r5,32($out) # Y
2252	std r6,40($out)
2253	std r7,48($out)
2254	std r8,56($out)
2255
2256	lwz r5, 64*0($inp)
2257	lwz r6, 64*1($inp)
2258	lwz r7, 64*2($inp)
2259	lwz r8, 64*3($inp)
2260	lwz r9, 64*4($inp)
2261	lwz r10,64*5($inp)
2262	lwz r11,64*6($inp)
2263	lwz r12,64*7($inp)
2264	sldi r9, r9, 32
2265	sldi r10,r10,32
2266	sldi r11,r11,32
2267	sldi r12,r12,32
2268	or r5,r5,r9
2269	or r6,r6,r10
2270	or r7,r7,r11
2271	or r8,r8,r12
2272	and r5,r5,r0
2273	and r6,r6,r0
2274	and r7,r7,r0
2275	and r8,r8,r0
2276	std r5,64($out) # Z
2277	std r6,72($out)
2278	std r7,80($out)
2279	std r8,88($out)
2280
2281	blr
2282	.long 0
2283	.byte 0,12,0x14,0,0,0,3,0
2284	.long 0
2285	.size ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
2286
2287	########################################################################
2288	# void ecp_nistz256_scatter_w7(void out, const P256_POINT_AFFINE inp,
2289	# int index);
2290	.globl ecp_nistz256_scatter_w7
2291	.align 4
2292	ecp_nistz256_scatter_w7:
2293	li r0,8
2294	mtctr r0
2295	add $out,$out,$index
2296	subi $inp,$inp,8
2297
2298	.Loop_scatter_w7:
2299	ldu r0,8($inp)
2300	stb r0,64*0($out)
2301	srdi r0,r0,8
2302	stb r0,64*1($out)
2303	srdi r0,r0,8
2304	stb r0,64*2($out)
2305	srdi r0,r0,8
2306	stb r0,64*3($out)
2307	srdi r0,r0,8
2308	stb r0,64*4($out)
2309	srdi r0,r0,8
2310	stb r0,64*5($out)
2311	srdi r0,r0,8
2312	stb r0,64*6($out)
2313	srdi r0,r0,8
2314	stb r0,64*7($out)
2315	addi $out,$out,64*8
2316	bdnz .Loop_scatter_w7
2317
2318	blr
2319	.long 0
2320	.byte 0,12,0x14,0,0,0,3,0
2321	.long 0
2322	.size ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
2323
2324	########################################################################
2325	# void ecp_nistz256_gather_w7(P256_POINT_AFFINE out, const void inp,
2326	# int index);
2327	.globl ecp_nistz256_gather_w7
2328	.align 4
2329	ecp_nistz256_gather_w7:
2330	li r0,8
2331	mtctr r0
2332	neg r0,$index
2333	sradi r0,r0,63
2334
2335	add $index,$index,r0
2336	add $inp,$inp,$index
2337	subi $out,$out,8
2338
2339	.Loop_gather_w7:
2340	lbz r5, 64*0($inp)
2341	lbz r6, 64*1($inp)
2342	lbz r7, 64*2($inp)
2343	lbz r8, 64*3($inp)
2344	lbz r9, 64*4($inp)
2345	lbz r10,64*5($inp)
2346	lbz r11,64*6($inp)
2347	lbz r12,64*7($inp)
2348	addi $inp,$inp,64*8
2349
2350	sldi r6, r6, 8
2351	sldi r7, r7, 16
2352	sldi r8, r8, 24
2353	sldi r9, r9, 32
2354	sldi r10,r10,40
2355	sldi r11,r11,48
2356	sldi r12,r12,56
2357
2358	or r5,r5,r6
2359	or r7,r7,r8
2360	or r9,r9,r10
2361	or r11,r11,r12
2362	or r5,r5,r7
2363	or r9,r9,r11
2364	or r5,r5,r9
2365	and r5,r5,r0
2366	stdu r5,8($out)
2367	bdnz .Loop_gather_w7
2368
2369	blr
2370	.long 0
2371	.byte 0,12,0x14,0,0,0,3,0
2372	.long 0
2373	.size ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
2374	___
2375	}
2376
2377	foreach (split("\n",$code)) {
2378	s/\`([^\`]*)\`/eval $1/ge;
2379
2380	print $_,"\n";
2381	}
2382	close STDOUT or die "error closing STDOUT: $!"; # enforce flush

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/libs/openssl-1.1.1k/crypto/ec/asm/ecp_nistz256-ppc64.pl@ 90293

Download in other formats: