aesp8-ppc.pl@ 69890

Last change on this file since 69890 was 69890, checked in by vboxsync, 7 years ago
Added OpenSSL 1.1.0g with unneeded files removed, otherwise unmodified. bugref:8070: src/libs maintenance
Property svn:eol-style set to `LF` Property svn:executable set to ``*
File size: 91.1 KB

Line
1	#! /usr/bin/env perl
2	# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
3	#
4	# Licensed under the OpenSSL license (the "License"). You may not use
5	# this file except in compliance with the License. You can obtain a copy
6	# in the file LICENSE in the source distribution or at
7	# https://www.openssl.org/source/license.html
8
9	#
10	# ====================================================================
11	# Written by Andy Polyakov <[email protected]> for the OpenSSL
12	# project. The module is, however, dual licensed under OpenSSL and
13	# CRYPTOGAMS licenses depending on where you obtain it. For further
14	# details see http://www.openssl.org/~appro/cryptogams/.
15	# ====================================================================
16	#
17	# This module implements support for AES instructions as per PowerISA
18	# specification version 2.07, first implemented by POWER8 processor.
19	# The module is endian-agnostic in sense that it supports both big-
20	# and little-endian cases. Data alignment in parallelizable modes is
21	# handled with VSX loads and stores, which implies MSR.VSX flag being
22	# set. It should also be noted that ISA specification doesn't prohibit
23	# alignment exceptions for these instructions on page boundaries.
24	# Initially alignment was handled in pure AltiVec/VMX way [when data
25	# is aligned programmatically, which in turn guarantees exception-
26	# free execution], but it turned to hamper performance when vcipher
27	# instructions are interleaved. It's reckoned that eventual
28	# misalignment penalties at page boundaries are in average lower
29	# than additional overhead in pure AltiVec approach.
30	#
31	# May 2016
32	#
33	# Add XTS subroutine, 9x on little- and 12x improvement on big-endian
34	# systems were measured.
35	#
36	######################################################################
37	# Current large-block performance in cycles per byte processed with
38	# 128-bit key (less is better).
39	#
40	# CBC en-/decrypt CTR XTS
41	# POWER8[le] 3.96/0.72 0.74 1.1
42	# POWER8[be] 3.75/0.65 0.66 1.0
43
44	$flavour = shift;
45
46	if ($flavour =~ /64/) {
47	$SIZE_T =8;
48	$LRSAVE =2*$SIZE_T;
49	$STU ="stdu";
50	$POP ="ld";
51	$PUSH ="std";
52	$UCMP ="cmpld";
53	$SHL ="sldi";
54	} elsif ($flavour =~ /32/) {
55	$SIZE_T =4;
56	$LRSAVE =$SIZE_T;
57	$STU ="stwu";
58	$POP ="lwz";
59	$PUSH ="stw";
60	$UCMP ="cmplw";
61	$SHL ="slwi";
62	} else { die "nonsense $flavour"; }
63
64	$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
65
66	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
67	( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
68	( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
69	die "can't locate ppc-xlate.pl";
70
71	open STDOUT,"\| $^X $xlate $flavour ".shift \|\| die "can't call $xlate: $!";
72
73	$FRAME=8*$SIZE_T;
74	$prefix="aes_p8";
75
76	$sp="r1";
77	$vrsave="r12";
78
79	#########################################################################
80	{{{ # Key setup procedures #
81	my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
82	my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
83	my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
84
85	$code.=<<___;
86	.machine "any"
87
88	.text
89
90	.align 7
91	rcon:
92	.long 0x01000000, 0x01000000, 0x01000000, 0x01000000 ?rev
93	.long 0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000 ?rev
94	.long 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c ?rev
95	.long 0,0,0,0 ?asis
96	Lconsts:
97	mflr r0
98	bcl 20,31,\$+4
99	mflr $ptr #vvvvv "distance between . and rcon
100	addi $ptr,$ptr,-0x48
101	mtlr r0
102	blr
103	.long 0
104	.byte 0,12,0x14,0,0,0,0,0
105	.asciz "AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
106
107	.globl .${prefix}_set_encrypt_key
108	.align 5
109	.${prefix}_set_encrypt_key:
110	Lset_encrypt_key:
111	mflr r11
112	$PUSH r11,$LRSAVE($sp)
113
114	li $ptr,-1
115	${UCMP}i $inp,0
116	beq- Lenc_key_abort # if ($inp==0) return -1;
117	${UCMP}i $out,0
118	beq- Lenc_key_abort # if ($out==0) return -1;
119	li $ptr,-2
120	cmpwi $bits,128
121	blt- Lenc_key_abort
122	cmpwi $bits,256
123	bgt- Lenc_key_abort
124	andi. r0,$bits,0x3f
125	bne- Lenc_key_abort
126
127	lis r0,0xfff0
128	mfspr $vrsave,256
129	mtspr 256,r0
130
131	bl Lconsts
132	mtlr r11
133
134	neg r9,$inp
135	lvx $in0,0,$inp
136	addi $inp,$inp,15 # 15 is not typo
137	lvsr $key,0,r9 # borrow $key
138	li r8,0x20
139	cmpwi $bits,192
140	lvx $in1,0,$inp
141	le?vspltisb $mask,0x0f # borrow $mask
142	lvx $rcon,0,$ptr
143	le?vxor $key,$key,$mask # adjust for byte swap
144	lvx $mask,r8,$ptr
145	addi $ptr,$ptr,0x10
146	vperm $in0,$in0,$in1,$key # align [and byte swap in LE]
147	li $cnt,8
148	vxor $zero,$zero,$zero
149	mtctr $cnt
150
151	?lvsr $outperm,0,$out
152	vspltisb $outmask,-1
153	lvx $outhead,0,$out
154	?vperm $outmask,$zero,$outmask,$outperm
155
156	blt Loop128
157	addi $inp,$inp,8
158	beq L192
159	addi $inp,$inp,8
160	b L256
161
162	.align 4
163	Loop128:
164	vperm $key,$in0,$in0,$mask # rotate-n-splat
165	vsldoi $tmp,$zero,$in0,12 # >>32
166	vperm $outtail,$in0,$in0,$outperm # rotate
167	vsel $stage,$outhead,$outtail,$outmask
168	vmr $outhead,$outtail
169	vcipherlast $key,$key,$rcon
170	stvx $stage,0,$out
171	addi $out,$out,16
172
173	vxor $in0,$in0,$tmp
174	vsldoi $tmp,$zero,$tmp,12 # >>32
175	vxor $in0,$in0,$tmp
176	vsldoi $tmp,$zero,$tmp,12 # >>32
177	vxor $in0,$in0,$tmp
178	vadduwm $rcon,$rcon,$rcon
179	vxor $in0,$in0,$key
180	bdnz Loop128
181
182	lvx $rcon,0,$ptr # last two round keys
183
184	vperm $key,$in0,$in0,$mask # rotate-n-splat
185	vsldoi $tmp,$zero,$in0,12 # >>32
186	vperm $outtail,$in0,$in0,$outperm # rotate
187	vsel $stage,$outhead,$outtail,$outmask
188	vmr $outhead,$outtail
189	vcipherlast $key,$key,$rcon
190	stvx $stage,0,$out
191	addi $out,$out,16
192
193	vxor $in0,$in0,$tmp
194	vsldoi $tmp,$zero,$tmp,12 # >>32
195	vxor $in0,$in0,$tmp
196	vsldoi $tmp,$zero,$tmp,12 # >>32
197	vxor $in0,$in0,$tmp
198	vadduwm $rcon,$rcon,$rcon
199	vxor $in0,$in0,$key
200
201	vperm $key,$in0,$in0,$mask # rotate-n-splat
202	vsldoi $tmp,$zero,$in0,12 # >>32
203	vperm $outtail,$in0,$in0,$outperm # rotate
204	vsel $stage,$outhead,$outtail,$outmask
205	vmr $outhead,$outtail
206	vcipherlast $key,$key,$rcon
207	stvx $stage,0,$out
208	addi $out,$out,16
209
210	vxor $in0,$in0,$tmp
211	vsldoi $tmp,$zero,$tmp,12 # >>32
212	vxor $in0,$in0,$tmp
213	vsldoi $tmp,$zero,$tmp,12 # >>32
214	vxor $in0,$in0,$tmp
215	vxor $in0,$in0,$key
216	vperm $outtail,$in0,$in0,$outperm # rotate
217	vsel $stage,$outhead,$outtail,$outmask
218	vmr $outhead,$outtail
219	stvx $stage,0,$out
220
221	addi $inp,$out,15 # 15 is not typo
222	addi $out,$out,0x50
223
224	li $rounds,10
225	b Ldone
226
227	.align 4
228	L192:
229	lvx $tmp,0,$inp
230	li $cnt,4
231	vperm $outtail,$in0,$in0,$outperm # rotate
232	vsel $stage,$outhead,$outtail,$outmask
233	vmr $outhead,$outtail
234	stvx $stage,0,$out
235	addi $out,$out,16
236	vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
237	vspltisb $key,8 # borrow $key
238	mtctr $cnt
239	vsububm $mask,$mask,$key # adjust the mask
240
241	Loop192:
242	vperm $key,$in1,$in1,$mask # roate-n-splat
243	vsldoi $tmp,$zero,$in0,12 # >>32
244	vcipherlast $key,$key,$rcon
245
246	vxor $in0,$in0,$tmp
247	vsldoi $tmp,$zero,$tmp,12 # >>32
248	vxor $in0,$in0,$tmp
249	vsldoi $tmp,$zero,$tmp,12 # >>32
250	vxor $in0,$in0,$tmp
251
252	vsldoi $stage,$zero,$in1,8
253	vspltw $tmp,$in0,3
254	vxor $tmp,$tmp,$in1
255	vsldoi $in1,$zero,$in1,12 # >>32
256	vadduwm $rcon,$rcon,$rcon
257	vxor $in1,$in1,$tmp
258	vxor $in0,$in0,$key
259	vxor $in1,$in1,$key
260	vsldoi $stage,$stage,$in0,8
261
262	vperm $key,$in1,$in1,$mask # rotate-n-splat
263	vsldoi $tmp,$zero,$in0,12 # >>32
264	vperm $outtail,$stage,$stage,$outperm # rotate
265	vsel $stage,$outhead,$outtail,$outmask
266	vmr $outhead,$outtail
267	vcipherlast $key,$key,$rcon
268	stvx $stage,0,$out
269	addi $out,$out,16
270
271	vsldoi $stage,$in0,$in1,8
272	vxor $in0,$in0,$tmp
273	vsldoi $tmp,$zero,$tmp,12 # >>32
274	vperm $outtail,$stage,$stage,$outperm # rotate
275	vsel $stage,$outhead,$outtail,$outmask
276	vmr $outhead,$outtail
277	vxor $in0,$in0,$tmp
278	vsldoi $tmp,$zero,$tmp,12 # >>32
279	vxor $in0,$in0,$tmp
280	stvx $stage,0,$out
281	addi $out,$out,16
282
283	vspltw $tmp,$in0,3
284	vxor $tmp,$tmp,$in1
285	vsldoi $in1,$zero,$in1,12 # >>32
286	vadduwm $rcon,$rcon,$rcon
287	vxor $in1,$in1,$tmp
288	vxor $in0,$in0,$key
289	vxor $in1,$in1,$key
290	vperm $outtail,$in0,$in0,$outperm # rotate
291	vsel $stage,$outhead,$outtail,$outmask
292	vmr $outhead,$outtail
293	stvx $stage,0,$out
294	addi $inp,$out,15 # 15 is not typo
295	addi $out,$out,16
296	bdnz Loop192
297
298	li $rounds,12
299	addi $out,$out,0x20
300	b Ldone
301
302	.align 4
303	L256:
304	lvx $tmp,0,$inp
305	li $cnt,7
306	li $rounds,14
307	vperm $outtail,$in0,$in0,$outperm # rotate
308	vsel $stage,$outhead,$outtail,$outmask
309	vmr $outhead,$outtail
310	stvx $stage,0,$out
311	addi $out,$out,16
312	vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
313	mtctr $cnt
314
315	Loop256:
316	vperm $key,$in1,$in1,$mask # rotate-n-splat
317	vsldoi $tmp,$zero,$in0,12 # >>32
318	vperm $outtail,$in1,$in1,$outperm # rotate
319	vsel $stage,$outhead,$outtail,$outmask
320	vmr $outhead,$outtail
321	vcipherlast $key,$key,$rcon
322	stvx $stage,0,$out
323	addi $out,$out,16
324
325	vxor $in0,$in0,$tmp
326	vsldoi $tmp,$zero,$tmp,12 # >>32
327	vxor $in0,$in0,$tmp
328	vsldoi $tmp,$zero,$tmp,12 # >>32
329	vxor $in0,$in0,$tmp
330	vadduwm $rcon,$rcon,$rcon
331	vxor $in0,$in0,$key
332	vperm $outtail,$in0,$in0,$outperm # rotate
333	vsel $stage,$outhead,$outtail,$outmask
334	vmr $outhead,$outtail
335	stvx $stage,0,$out
336	addi $inp,$out,15 # 15 is not typo
337	addi $out,$out,16
338	bdz Ldone
339
340	vspltw $key,$in0,3 # just splat
341	vsldoi $tmp,$zero,$in1,12 # >>32
342	vsbox $key,$key
343
344	vxor $in1,$in1,$tmp
345	vsldoi $tmp,$zero,$tmp,12 # >>32
346	vxor $in1,$in1,$tmp
347	vsldoi $tmp,$zero,$tmp,12 # >>32
348	vxor $in1,$in1,$tmp
349
350	vxor $in1,$in1,$key
351	b Loop256
352
353	.align 4
354	Ldone:
355	lvx $in1,0,$inp # redundant in aligned case
356	vsel $in1,$outhead,$in1,$outmask
357	stvx $in1,0,$inp
358	li $ptr,0
359	mtspr 256,$vrsave
360	stw $rounds,0($out)
361
362	Lenc_key_abort:
363	mr r3,$ptr
364	blr
365	.long 0
366	.byte 0,12,0x14,1,0,0,3,0
367	.long 0
368	.size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
369
370	.globl .${prefix}_set_decrypt_key
371	.align 5
372	.${prefix}_set_decrypt_key:
373	$STU $sp,-$FRAME($sp)
374	mflr r10
375	$PUSH r10,$FRAME+$LRSAVE($sp)
376	bl Lset_encrypt_key
377	mtlr r10
378
379	cmpwi r3,0
380	bne- Ldec_key_abort
381
382	slwi $cnt,$rounds,4
383	subi $inp,$out,240 # first round key
384	srwi $rounds,$rounds,1
385	add $out,$inp,$cnt # last round key
386	mtctr $rounds
387
388	Ldeckey:
389	lwz r0, 0($inp)
390	lwz r6, 4($inp)
391	lwz r7, 8($inp)
392	lwz r8, 12($inp)
393	addi $inp,$inp,16
394	lwz r9, 0($out)
395	lwz r10,4($out)
396	lwz r11,8($out)
397	lwz r12,12($out)
398	stw r0, 0($out)
399	stw r6, 4($out)
400	stw r7, 8($out)
401	stw r8, 12($out)
402	subi $out,$out,16
403	stw r9, -16($inp)
404	stw r10,-12($inp)
405	stw r11,-8($inp)
406	stw r12,-4($inp)
407	bdnz Ldeckey
408
409	xor r3,r3,r3 # return value
410	Ldec_key_abort:
411	addi $sp,$sp,$FRAME
412	blr
413	.long 0
414	.byte 0,12,4,1,0x80,0,3,0
415	.long 0
416	.size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
417	___
418	}}}
419	#########################################################################
420	{{{ # Single block en- and decrypt procedures #
421	sub gen_block () {
422	my $dir = shift;
423	my $n = $dir eq "de" ? "n" : "";
424	my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
425
426	$code.=<<___;
427	.globl .${prefix}_${dir}crypt
428	.align 5
429	.${prefix}_${dir}crypt:
430	lwz $rounds,240($key)
431	lis r0,0xfc00
432	mfspr $vrsave,256
433	li $idx,15 # 15 is not typo
434	mtspr 256,r0
435
436	lvx v0,0,$inp
437	neg r11,$out
438	lvx v1,$idx,$inp
439	lvsl v2,0,$inp # inpperm
440	le?vspltisb v4,0x0f
441	?lvsl v3,0,r11 # outperm
442	le?vxor v2,v2,v4
443	li $idx,16
444	vperm v0,v0,v1,v2 # align [and byte swap in LE]
445	lvx v1,0,$key
446	?lvsl v5,0,$key # keyperm
447	srwi $rounds,$rounds,1
448	lvx v2,$idx,$key
449	addi $idx,$idx,16
450	subi $rounds,$rounds,1
451	?vperm v1,v1,v2,v5 # align round key
452
453	vxor v0,v0,v1
454	lvx v1,$idx,$key
455	addi $idx,$idx,16
456	mtctr $rounds
457
458	Loop_${dir}c:
459	?vperm v2,v2,v1,v5
460	v${n}cipher v0,v0,v2
461	lvx v2,$idx,$key
462	addi $idx,$idx,16
463	?vperm v1,v1,v2,v5
464	v${n}cipher v0,v0,v1
465	lvx v1,$idx,$key
466	addi $idx,$idx,16
467	bdnz Loop_${dir}c
468
469	?vperm v2,v2,v1,v5
470	v${n}cipher v0,v0,v2
471	lvx v2,$idx,$key
472	?vperm v1,v1,v2,v5
473	v${n}cipherlast v0,v0,v1
474
475	vspltisb v2,-1
476	vxor v1,v1,v1
477	li $idx,15 # 15 is not typo
478	?vperm v2,v1,v2,v3 # outmask
479	le?vxor v3,v3,v4
480	lvx v1,0,$out # outhead
481	vperm v0,v0,v0,v3 # rotate [and byte swap in LE]
482	vsel v1,v1,v0,v2
483	lvx v4,$idx,$out
484	stvx v1,0,$out
485	vsel v0,v0,v4,v2
486	stvx v0,$idx,$out
487
488	mtspr 256,$vrsave
489	blr
490	.long 0
491	.byte 0,12,0x14,0,0,0,3,0
492	.long 0
493	.size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
494	___
495	}
496	&gen_block("en");
497	&gen_block("de");
498	}}}
499	#########################################################################
500	{{{ # CBC en- and decrypt procedures #
501	my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
502	my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
503	my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
504	map("v$_",(4..10));
505	$code.=<<___;
506	.globl .${prefix}_cbc_encrypt
507	.align 5
508	.${prefix}_cbc_encrypt:
509	${UCMP}i $len,16
510	bltlr-
511
512	cmpwi $enc,0 # test direction
513	lis r0,0xffe0
514	mfspr $vrsave,256
515	mtspr 256,r0
516
517	li $idx,15
518	vxor $rndkey0,$rndkey0,$rndkey0
519	le?vspltisb $tmp,0x0f
520
521	lvx $ivec,0,$ivp # load [unaligned] iv
522	lvsl $inpperm,0,$ivp
523	lvx $inptail,$idx,$ivp
524	le?vxor $inpperm,$inpperm,$tmp
525	vperm $ivec,$ivec,$inptail,$inpperm
526
527	neg r11,$inp
528	?lvsl $keyperm,0,$key # prepare for unaligned key
529	lwz $rounds,240($key)
530
531	lvsr $inpperm,0,r11 # prepare for unaligned load
532	lvx $inptail,0,$inp
533	addi $inp,$inp,15 # 15 is not typo
534	le?vxor $inpperm,$inpperm,$tmp
535
536	?lvsr $outperm,0,$out # prepare for unaligned store
537	vspltisb $outmask,-1
538	lvx $outhead,0,$out
539	?vperm $outmask,$rndkey0,$outmask,$outperm
540	le?vxor $outperm,$outperm,$tmp
541
542	srwi $rounds,$rounds,1
543	li $idx,16
544	subi $rounds,$rounds,1
545	beq Lcbc_dec
546
547	Lcbc_enc:
548	vmr $inout,$inptail
549	lvx $inptail,0,$inp
550	addi $inp,$inp,16
551	mtctr $rounds
552	subi $len,$len,16 # len-=16
553
554	lvx $rndkey0,0,$key
555	vperm $inout,$inout,$inptail,$inpperm
556	lvx $rndkey1,$idx,$key
557	addi $idx,$idx,16
558	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
559	vxor $inout,$inout,$rndkey0
560	lvx $rndkey0,$idx,$key
561	addi $idx,$idx,16
562	vxor $inout,$inout,$ivec
563
564	Loop_cbc_enc:
565	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
566	vcipher $inout,$inout,$rndkey1
567	lvx $rndkey1,$idx,$key
568	addi $idx,$idx,16
569	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
570	vcipher $inout,$inout,$rndkey0
571	lvx $rndkey0,$idx,$key
572	addi $idx,$idx,16
573	bdnz Loop_cbc_enc
574
575	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
576	vcipher $inout,$inout,$rndkey1
577	lvx $rndkey1,$idx,$key
578	li $idx,16
579	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
580	vcipherlast $ivec,$inout,$rndkey0
581	${UCMP}i $len,16
582
583	vperm $tmp,$ivec,$ivec,$outperm
584	vsel $inout,$outhead,$tmp,$outmask
585	vmr $outhead,$tmp
586	stvx $inout,0,$out
587	addi $out,$out,16
588	bge Lcbc_enc
589
590	b Lcbc_done
591
592	.align 4
593	Lcbc_dec:
594	${UCMP}i $len,128
595	bge _aesp8_cbc_decrypt8x
596	vmr $tmp,$inptail
597	lvx $inptail,0,$inp
598	addi $inp,$inp,16
599	mtctr $rounds
600	subi $len,$len,16 # len-=16
601
602	lvx $rndkey0,0,$key
603	vperm $tmp,$tmp,$inptail,$inpperm
604	lvx $rndkey1,$idx,$key
605	addi $idx,$idx,16
606	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
607	vxor $inout,$tmp,$rndkey0
608	lvx $rndkey0,$idx,$key
609	addi $idx,$idx,16
610
611	Loop_cbc_dec:
612	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
613	vncipher $inout,$inout,$rndkey1
614	lvx $rndkey1,$idx,$key
615	addi $idx,$idx,16
616	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
617	vncipher $inout,$inout,$rndkey0
618	lvx $rndkey0,$idx,$key
619	addi $idx,$idx,16
620	bdnz Loop_cbc_dec
621
622	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
623	vncipher $inout,$inout,$rndkey1
624	lvx $rndkey1,$idx,$key
625	li $idx,16
626	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
627	vncipherlast $inout,$inout,$rndkey0
628	${UCMP}i $len,16
629
630	vxor $inout,$inout,$ivec
631	vmr $ivec,$tmp
632	vperm $tmp,$inout,$inout,$outperm
633	vsel $inout,$outhead,$tmp,$outmask
634	vmr $outhead,$tmp
635	stvx $inout,0,$out
636	addi $out,$out,16
637	bge Lcbc_dec
638
639	Lcbc_done:
640	addi $out,$out,-1
641	lvx $inout,0,$out # redundant in aligned case
642	vsel $inout,$outhead,$inout,$outmask
643	stvx $inout,0,$out
644
645	neg $enc,$ivp # write [unaligned] iv
646	li $idx,15 # 15 is not typo
647	vxor $rndkey0,$rndkey0,$rndkey0
648	vspltisb $outmask,-1
649	le?vspltisb $tmp,0x0f
650	?lvsl $outperm,0,$enc
651	?vperm $outmask,$rndkey0,$outmask,$outperm
652	le?vxor $outperm,$outperm,$tmp
653	lvx $outhead,0,$ivp
654	vperm $ivec,$ivec,$ivec,$outperm
655	vsel $inout,$outhead,$ivec,$outmask
656	lvx $inptail,$idx,$ivp
657	stvx $inout,0,$ivp
658	vsel $inout,$ivec,$inptail,$outmask
659	stvx $inout,$idx,$ivp
660
661	mtspr 256,$vrsave
662	blr
663	.long 0
664	.byte 0,12,0x14,0,0,0,6,0
665	.long 0
666	___
667	#########################################################################
668	{{ # Optimized CBC decrypt procedure #
669	my $key_="r11";
670	my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
671	$x00=0 if ($flavour =~ /osx/);
672	my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
673	my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
674	my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
675	# v26-v31 last 6 round keys
676	my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
677
678	$code.=<<___;
679	.align 5
680	_aesp8_cbc_decrypt8x:
681	$STU $sp,-`($FRAME+2116+6$SIZE_T)`($sp)
682	li r10,`$FRAME+8*16+15`
683	li r11,`$FRAME+8*16+31`
684	stvx v20,r10,$sp # ABI says so
685	addi r10,r10,32
686	stvx v21,r11,$sp
687	addi r11,r11,32
688	stvx v22,r10,$sp
689	addi r10,r10,32
690	stvx v23,r11,$sp
691	addi r11,r11,32
692	stvx v24,r10,$sp
693	addi r10,r10,32
694	stvx v25,r11,$sp
695	addi r11,r11,32
696	stvx v26,r10,$sp
697	addi r10,r10,32
698	stvx v27,r11,$sp
699	addi r11,r11,32
700	stvx v28,r10,$sp
701	addi r10,r10,32
702	stvx v29,r11,$sp
703	addi r11,r11,32
704	stvx v30,r10,$sp
705	stvx v31,r11,$sp
706	li r0,-1
707	stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
708	li $x10,0x10
709	$PUSH r26,`$FRAME+2116+0$SIZE_T`($sp)
710	li $x20,0x20
711	$PUSH r27,`$FRAME+2116+1$SIZE_T`($sp)
712	li $x30,0x30
713	$PUSH r28,`$FRAME+2116+2$SIZE_T`($sp)
714	li $x40,0x40
715	$PUSH r29,`$FRAME+2116+3$SIZE_T`($sp)
716	li $x50,0x50
717	$PUSH r30,`$FRAME+2116+4$SIZE_T`($sp)
718	li $x60,0x60
719	$PUSH r31,`$FRAME+2116+5$SIZE_T`($sp)
720	li $x70,0x70
721	mtspr 256,r0
722
723	subi $rounds,$rounds,3 # -4 in total
724	subi $len,$len,128 # bias
725
726	lvx $rndkey0,$x00,$key # load key schedule
727	lvx v30,$x10,$key
728	addi $key,$key,0x20
729	lvx v31,$x00,$key
730	?vperm $rndkey0,$rndkey0,v30,$keyperm
731	addi $key_,$sp,$FRAME+15
732	mtctr $rounds
733
734	Load_cbc_dec_key:
735	?vperm v24,v30,v31,$keyperm
736	lvx v30,$x10,$key
737	addi $key,$key,0x20
738	stvx v24,$x00,$key_ # off-load round[1]
739	?vperm v25,v31,v30,$keyperm
740	lvx v31,$x00,$key
741	stvx v25,$x10,$key_ # off-load round[2]
742	addi $key_,$key_,0x20
743	bdnz Load_cbc_dec_key
744
745	lvx v26,$x10,$key
746	?vperm v24,v30,v31,$keyperm
747	lvx v27,$x20,$key
748	stvx v24,$x00,$key_ # off-load round[3]
749	?vperm v25,v31,v26,$keyperm
750	lvx v28,$x30,$key
751	stvx v25,$x10,$key_ # off-load round[4]
752	addi $key_,$sp,$FRAME+15 # rewind $key_
753	?vperm v26,v26,v27,$keyperm
754	lvx v29,$x40,$key
755	?vperm v27,v27,v28,$keyperm
756	lvx v30,$x50,$key
757	?vperm v28,v28,v29,$keyperm
758	lvx v31,$x60,$key
759	?vperm v29,v29,v30,$keyperm
760	lvx $out0,$x70,$key # borrow $out0
761	?vperm v30,v30,v31,$keyperm
762	lvx v24,$x00,$key_ # pre-load round[1]
763	?vperm v31,v31,$out0,$keyperm
764	lvx v25,$x10,$key_ # pre-load round[2]
765
766	#lvx $inptail,0,$inp # "caller" already did this
767	#addi $inp,$inp,15 # 15 is not typo
768	subi $inp,$inp,15 # undo "caller"
769
770	le?li $idx,8
771	lvx_u $in0,$x00,$inp # load first 8 "words"
772	le?lvsl $inpperm,0,$idx
773	le?vspltisb $tmp,0x0f
774	lvx_u $in1,$x10,$inp
775	le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
776	lvx_u $in2,$x20,$inp
777	le?vperm $in0,$in0,$in0,$inpperm
778	lvx_u $in3,$x30,$inp
779	le?vperm $in1,$in1,$in1,$inpperm
780	lvx_u $in4,$x40,$inp
781	le?vperm $in2,$in2,$in2,$inpperm
782	vxor $out0,$in0,$rndkey0
783	lvx_u $in5,$x50,$inp
784	le?vperm $in3,$in3,$in3,$inpperm
785	vxor $out1,$in1,$rndkey0
786	lvx_u $in6,$x60,$inp
787	le?vperm $in4,$in4,$in4,$inpperm
788	vxor $out2,$in2,$rndkey0
789	lvx_u $in7,$x70,$inp
790	addi $inp,$inp,0x80
791	le?vperm $in5,$in5,$in5,$inpperm
792	vxor $out3,$in3,$rndkey0
793	le?vperm $in6,$in6,$in6,$inpperm
794	vxor $out4,$in4,$rndkey0
795	le?vperm $in7,$in7,$in7,$inpperm
796	vxor $out5,$in5,$rndkey0
797	vxor $out6,$in6,$rndkey0
798	vxor $out7,$in7,$rndkey0
799
800	mtctr $rounds
801	b Loop_cbc_dec8x
802	.align 5
803	Loop_cbc_dec8x:
804	vncipher $out0,$out0,v24
805	vncipher $out1,$out1,v24
806	vncipher $out2,$out2,v24
807	vncipher $out3,$out3,v24
808	vncipher $out4,$out4,v24
809	vncipher $out5,$out5,v24
810	vncipher $out6,$out6,v24
811	vncipher $out7,$out7,v24
812	lvx v24,$x20,$key_ # round[3]
813	addi $key_,$key_,0x20
814
815	vncipher $out0,$out0,v25
816	vncipher $out1,$out1,v25
817	vncipher $out2,$out2,v25
818	vncipher $out3,$out3,v25
819	vncipher $out4,$out4,v25
820	vncipher $out5,$out5,v25
821	vncipher $out6,$out6,v25
822	vncipher $out7,$out7,v25
823	lvx v25,$x10,$key_ # round[4]
824	bdnz Loop_cbc_dec8x
825
826	subic $len,$len,128 # $len-=128
827	vncipher $out0,$out0,v24
828	vncipher $out1,$out1,v24
829	vncipher $out2,$out2,v24
830	vncipher $out3,$out3,v24
831	vncipher $out4,$out4,v24
832	vncipher $out5,$out5,v24
833	vncipher $out6,$out6,v24
834	vncipher $out7,$out7,v24
835
836	subfe. r0,r0,r0 # borrow?-1:0
837	vncipher $out0,$out0,v25
838	vncipher $out1,$out1,v25
839	vncipher $out2,$out2,v25
840	vncipher $out3,$out3,v25
841	vncipher $out4,$out4,v25
842	vncipher $out5,$out5,v25
843	vncipher $out6,$out6,v25
844	vncipher $out7,$out7,v25
845
846	and r0,r0,$len
847	vncipher $out0,$out0,v26
848	vncipher $out1,$out1,v26
849	vncipher $out2,$out2,v26
850	vncipher $out3,$out3,v26
851	vncipher $out4,$out4,v26
852	vncipher $out5,$out5,v26
853	vncipher $out6,$out6,v26
854	vncipher $out7,$out7,v26
855
856	add $inp,$inp,r0 # $inp is adjusted in such
857	# way that at exit from the
858	# loop inX-in7 are loaded
859	# with last "words"
860	vncipher $out0,$out0,v27
861	vncipher $out1,$out1,v27
862	vncipher $out2,$out2,v27
863	vncipher $out3,$out3,v27
864	vncipher $out4,$out4,v27
865	vncipher $out5,$out5,v27
866	vncipher $out6,$out6,v27
867	vncipher $out7,$out7,v27
868
869	addi $key_,$sp,$FRAME+15 # rewind $key_
870	vncipher $out0,$out0,v28
871	vncipher $out1,$out1,v28
872	vncipher $out2,$out2,v28
873	vncipher $out3,$out3,v28
874	vncipher $out4,$out4,v28
875	vncipher $out5,$out5,v28
876	vncipher $out6,$out6,v28
877	vncipher $out7,$out7,v28
878	lvx v24,$x00,$key_ # re-pre-load round[1]
879
880	vncipher $out0,$out0,v29
881	vncipher $out1,$out1,v29
882	vncipher $out2,$out2,v29
883	vncipher $out3,$out3,v29
884	vncipher $out4,$out4,v29
885	vncipher $out5,$out5,v29
886	vncipher $out6,$out6,v29
887	vncipher $out7,$out7,v29
888	lvx v25,$x10,$key_ # re-pre-load round[2]
889
890	vncipher $out0,$out0,v30
891	vxor $ivec,$ivec,v31 # xor with last round key
892	vncipher $out1,$out1,v30
893	vxor $in0,$in0,v31
894	vncipher $out2,$out2,v30
895	vxor $in1,$in1,v31
896	vncipher $out3,$out3,v30
897	vxor $in2,$in2,v31
898	vncipher $out4,$out4,v30
899	vxor $in3,$in3,v31
900	vncipher $out5,$out5,v30
901	vxor $in4,$in4,v31
902	vncipher $out6,$out6,v30
903	vxor $in5,$in5,v31
904	vncipher $out7,$out7,v30
905	vxor $in6,$in6,v31
906
907	vncipherlast $out0,$out0,$ivec
908	vncipherlast $out1,$out1,$in0
909	lvx_u $in0,$x00,$inp # load next input block
910	vncipherlast $out2,$out2,$in1
911	lvx_u $in1,$x10,$inp
912	vncipherlast $out3,$out3,$in2
913	le?vperm $in0,$in0,$in0,$inpperm
914	lvx_u $in2,$x20,$inp
915	vncipherlast $out4,$out4,$in3
916	le?vperm $in1,$in1,$in1,$inpperm
917	lvx_u $in3,$x30,$inp
918	vncipherlast $out5,$out5,$in4
919	le?vperm $in2,$in2,$in2,$inpperm
920	lvx_u $in4,$x40,$inp
921	vncipherlast $out6,$out6,$in5
922	le?vperm $in3,$in3,$in3,$inpperm
923	lvx_u $in5,$x50,$inp
924	vncipherlast $out7,$out7,$in6
925	le?vperm $in4,$in4,$in4,$inpperm
926	lvx_u $in6,$x60,$inp
927	vmr $ivec,$in7
928	le?vperm $in5,$in5,$in5,$inpperm
929	lvx_u $in7,$x70,$inp
930	addi $inp,$inp,0x80
931
932	le?vperm $out0,$out0,$out0,$inpperm
933	le?vperm $out1,$out1,$out1,$inpperm
934	stvx_u $out0,$x00,$out
935	le?vperm $in6,$in6,$in6,$inpperm
936	vxor $out0,$in0,$rndkey0
937	le?vperm $out2,$out2,$out2,$inpperm
938	stvx_u $out1,$x10,$out
939	le?vperm $in7,$in7,$in7,$inpperm
940	vxor $out1,$in1,$rndkey0
941	le?vperm $out3,$out3,$out3,$inpperm
942	stvx_u $out2,$x20,$out
943	vxor $out2,$in2,$rndkey0
944	le?vperm $out4,$out4,$out4,$inpperm
945	stvx_u $out3,$x30,$out
946	vxor $out3,$in3,$rndkey0
947	le?vperm $out5,$out5,$out5,$inpperm
948	stvx_u $out4,$x40,$out
949	vxor $out4,$in4,$rndkey0
950	le?vperm $out6,$out6,$out6,$inpperm
951	stvx_u $out5,$x50,$out
952	vxor $out5,$in5,$rndkey0
953	le?vperm $out7,$out7,$out7,$inpperm
954	stvx_u $out6,$x60,$out
955	vxor $out6,$in6,$rndkey0
956	stvx_u $out7,$x70,$out
957	addi $out,$out,0x80
958	vxor $out7,$in7,$rndkey0
959
960	mtctr $rounds
961	beq Loop_cbc_dec8x # did $len-=128 borrow?
962
963	addic. $len,$len,128
964	beq Lcbc_dec8x_done
965	nop
966	nop
967
968	Loop_cbc_dec8x_tail: # up to 7 "words" tail...
969	vncipher $out1,$out1,v24
970	vncipher $out2,$out2,v24
971	vncipher $out3,$out3,v24
972	vncipher $out4,$out4,v24
973	vncipher $out5,$out5,v24
974	vncipher $out6,$out6,v24
975	vncipher $out7,$out7,v24
976	lvx v24,$x20,$key_ # round[3]
977	addi $key_,$key_,0x20
978
979	vncipher $out1,$out1,v25
980	vncipher $out2,$out2,v25
981	vncipher $out3,$out3,v25
982	vncipher $out4,$out4,v25
983	vncipher $out5,$out5,v25
984	vncipher $out6,$out6,v25
985	vncipher $out7,$out7,v25
986	lvx v25,$x10,$key_ # round[4]
987	bdnz Loop_cbc_dec8x_tail
988
989	vncipher $out1,$out1,v24
990	vncipher $out2,$out2,v24
991	vncipher $out3,$out3,v24
992	vncipher $out4,$out4,v24
993	vncipher $out5,$out5,v24
994	vncipher $out6,$out6,v24
995	vncipher $out7,$out7,v24
996
997	vncipher $out1,$out1,v25
998	vncipher $out2,$out2,v25
999	vncipher $out3,$out3,v25
1000	vncipher $out4,$out4,v25
1001	vncipher $out5,$out5,v25
1002	vncipher $out6,$out6,v25
1003	vncipher $out7,$out7,v25
1004
1005	vncipher $out1,$out1,v26
1006	vncipher $out2,$out2,v26
1007	vncipher $out3,$out3,v26
1008	vncipher $out4,$out4,v26
1009	vncipher $out5,$out5,v26
1010	vncipher $out6,$out6,v26
1011	vncipher $out7,$out7,v26
1012
1013	vncipher $out1,$out1,v27
1014	vncipher $out2,$out2,v27
1015	vncipher $out3,$out3,v27
1016	vncipher $out4,$out4,v27
1017	vncipher $out5,$out5,v27
1018	vncipher $out6,$out6,v27
1019	vncipher $out7,$out7,v27
1020
1021	vncipher $out1,$out1,v28
1022	vncipher $out2,$out2,v28
1023	vncipher $out3,$out3,v28
1024	vncipher $out4,$out4,v28
1025	vncipher $out5,$out5,v28
1026	vncipher $out6,$out6,v28
1027	vncipher $out7,$out7,v28
1028
1029	vncipher $out1,$out1,v29
1030	vncipher $out2,$out2,v29
1031	vncipher $out3,$out3,v29
1032	vncipher $out4,$out4,v29
1033	vncipher $out5,$out5,v29
1034	vncipher $out6,$out6,v29
1035	vncipher $out7,$out7,v29
1036
1037	vncipher $out1,$out1,v30
1038	vxor $ivec,$ivec,v31 # last round key
1039	vncipher $out2,$out2,v30
1040	vxor $in1,$in1,v31
1041	vncipher $out3,$out3,v30
1042	vxor $in2,$in2,v31
1043	vncipher $out4,$out4,v30
1044	vxor $in3,$in3,v31
1045	vncipher $out5,$out5,v30
1046	vxor $in4,$in4,v31
1047	vncipher $out6,$out6,v30
1048	vxor $in5,$in5,v31
1049	vncipher $out7,$out7,v30
1050	vxor $in6,$in6,v31
1051
1052	cmplwi $len,32 # switch($len)
1053	blt Lcbc_dec8x_one
1054	nop
1055	beq Lcbc_dec8x_two
1056	cmplwi $len,64
1057	blt Lcbc_dec8x_three
1058	nop
1059	beq Lcbc_dec8x_four
1060	cmplwi $len,96
1061	blt Lcbc_dec8x_five
1062	nop
1063	beq Lcbc_dec8x_six
1064
1065	Lcbc_dec8x_seven:
1066	vncipherlast $out1,$out1,$ivec
1067	vncipherlast $out2,$out2,$in1
1068	vncipherlast $out3,$out3,$in2
1069	vncipherlast $out4,$out4,$in3
1070	vncipherlast $out5,$out5,$in4
1071	vncipherlast $out6,$out6,$in5
1072	vncipherlast $out7,$out7,$in6
1073	vmr $ivec,$in7
1074
1075	le?vperm $out1,$out1,$out1,$inpperm
1076	le?vperm $out2,$out2,$out2,$inpperm
1077	stvx_u $out1,$x00,$out
1078	le?vperm $out3,$out3,$out3,$inpperm
1079	stvx_u $out2,$x10,$out
1080	le?vperm $out4,$out4,$out4,$inpperm
1081	stvx_u $out3,$x20,$out
1082	le?vperm $out5,$out5,$out5,$inpperm
1083	stvx_u $out4,$x30,$out
1084	le?vperm $out6,$out6,$out6,$inpperm
1085	stvx_u $out5,$x40,$out
1086	le?vperm $out7,$out7,$out7,$inpperm
1087	stvx_u $out6,$x50,$out
1088	stvx_u $out7,$x60,$out
1089	addi $out,$out,0x70
1090	b Lcbc_dec8x_done
1091
1092	.align 5
1093	Lcbc_dec8x_six:
1094	vncipherlast $out2,$out2,$ivec
1095	vncipherlast $out3,$out3,$in2
1096	vncipherlast $out4,$out4,$in3
1097	vncipherlast $out5,$out5,$in4
1098	vncipherlast $out6,$out6,$in5
1099	vncipherlast $out7,$out7,$in6
1100	vmr $ivec,$in7
1101
1102	le?vperm $out2,$out2,$out2,$inpperm
1103	le?vperm $out3,$out3,$out3,$inpperm
1104	stvx_u $out2,$x00,$out
1105	le?vperm $out4,$out4,$out4,$inpperm
1106	stvx_u $out3,$x10,$out
1107	le?vperm $out5,$out5,$out5,$inpperm
1108	stvx_u $out4,$x20,$out
1109	le?vperm $out6,$out6,$out6,$inpperm
1110	stvx_u $out5,$x30,$out
1111	le?vperm $out7,$out7,$out7,$inpperm
1112	stvx_u $out6,$x40,$out
1113	stvx_u $out7,$x50,$out
1114	addi $out,$out,0x60
1115	b Lcbc_dec8x_done
1116
1117	.align 5
1118	Lcbc_dec8x_five:
1119	vncipherlast $out3,$out3,$ivec
1120	vncipherlast $out4,$out4,$in3
1121	vncipherlast $out5,$out5,$in4
1122	vncipherlast $out6,$out6,$in5
1123	vncipherlast $out7,$out7,$in6
1124	vmr $ivec,$in7
1125
1126	le?vperm $out3,$out3,$out3,$inpperm
1127	le?vperm $out4,$out4,$out4,$inpperm
1128	stvx_u $out3,$x00,$out
1129	le?vperm $out5,$out5,$out5,$inpperm
1130	stvx_u $out4,$x10,$out
1131	le?vperm $out6,$out6,$out6,$inpperm
1132	stvx_u $out5,$x20,$out
1133	le?vperm $out7,$out7,$out7,$inpperm
1134	stvx_u $out6,$x30,$out
1135	stvx_u $out7,$x40,$out
1136	addi $out,$out,0x50
1137	b Lcbc_dec8x_done
1138
1139	.align 5
1140	Lcbc_dec8x_four:
1141	vncipherlast $out4,$out4,$ivec
1142	vncipherlast $out5,$out5,$in4
1143	vncipherlast $out6,$out6,$in5
1144	vncipherlast $out7,$out7,$in6
1145	vmr $ivec,$in7
1146
1147	le?vperm $out4,$out4,$out4,$inpperm
1148	le?vperm $out5,$out5,$out5,$inpperm
1149	stvx_u $out4,$x00,$out
1150	le?vperm $out6,$out6,$out6,$inpperm
1151	stvx_u $out5,$x10,$out
1152	le?vperm $out7,$out7,$out7,$inpperm
1153	stvx_u $out6,$x20,$out
1154	stvx_u $out7,$x30,$out
1155	addi $out,$out,0x40
1156	b Lcbc_dec8x_done
1157
1158	.align 5
1159	Lcbc_dec8x_three:
1160	vncipherlast $out5,$out5,$ivec
1161	vncipherlast $out6,$out6,$in5
1162	vncipherlast $out7,$out7,$in6
1163	vmr $ivec,$in7
1164
1165	le?vperm $out5,$out5,$out5,$inpperm
1166	le?vperm $out6,$out6,$out6,$inpperm
1167	stvx_u $out5,$x00,$out
1168	le?vperm $out7,$out7,$out7,$inpperm
1169	stvx_u $out6,$x10,$out
1170	stvx_u $out7,$x20,$out
1171	addi $out,$out,0x30
1172	b Lcbc_dec8x_done
1173
1174	.align 5
1175	Lcbc_dec8x_two:
1176	vncipherlast $out6,$out6,$ivec
1177	vncipherlast $out7,$out7,$in6
1178	vmr $ivec,$in7
1179
1180	le?vperm $out6,$out6,$out6,$inpperm
1181	le?vperm $out7,$out7,$out7,$inpperm
1182	stvx_u $out6,$x00,$out
1183	stvx_u $out7,$x10,$out
1184	addi $out,$out,0x20
1185	b Lcbc_dec8x_done
1186
1187	.align 5
1188	Lcbc_dec8x_one:
1189	vncipherlast $out7,$out7,$ivec
1190	vmr $ivec,$in7
1191
1192	le?vperm $out7,$out7,$out7,$inpperm
1193	stvx_u $out7,0,$out
1194	addi $out,$out,0x10
1195
1196	Lcbc_dec8x_done:
1197	le?vperm $ivec,$ivec,$ivec,$inpperm
1198	stvx_u $ivec,0,$ivp # write [unaligned] iv
1199
1200	li r10,`$FRAME+15`
1201	li r11,`$FRAME+31`
1202	stvx $inpperm,r10,$sp # wipe copies of round keys
1203	addi r10,r10,32
1204	stvx $inpperm,r11,$sp
1205	addi r11,r11,32
1206	stvx $inpperm,r10,$sp
1207	addi r10,r10,32
1208	stvx $inpperm,r11,$sp
1209	addi r11,r11,32
1210	stvx $inpperm,r10,$sp
1211	addi r10,r10,32
1212	stvx $inpperm,r11,$sp
1213	addi r11,r11,32
1214	stvx $inpperm,r10,$sp
1215	addi r10,r10,32
1216	stvx $inpperm,r11,$sp
1217	addi r11,r11,32
1218
1219	mtspr 256,$vrsave
1220	lvx v20,r10,$sp # ABI says so
1221	addi r10,r10,32
1222	lvx v21,r11,$sp
1223	addi r11,r11,32
1224	lvx v22,r10,$sp
1225	addi r10,r10,32
1226	lvx v23,r11,$sp
1227	addi r11,r11,32
1228	lvx v24,r10,$sp
1229	addi r10,r10,32
1230	lvx v25,r11,$sp
1231	addi r11,r11,32
1232	lvx v26,r10,$sp
1233	addi r10,r10,32
1234	lvx v27,r11,$sp
1235	addi r11,r11,32
1236	lvx v28,r10,$sp
1237	addi r10,r10,32
1238	lvx v29,r11,$sp
1239	addi r11,r11,32
1240	lvx v30,r10,$sp
1241	lvx v31,r11,$sp
1242	$POP r26,`$FRAME+2116+0$SIZE_T`($sp)
1243	$POP r27,`$FRAME+2116+1$SIZE_T`($sp)
1244	$POP r28,`$FRAME+2116+2$SIZE_T`($sp)
1245	$POP r29,`$FRAME+2116+3$SIZE_T`($sp)
1246	$POP r30,`$FRAME+2116+4$SIZE_T`($sp)
1247	$POP r31,`$FRAME+2116+5$SIZE_T`($sp)
1248	addi $sp,$sp,`$FRAME+2116+6$SIZE_T`
1249	blr
1250	.long 0
1251	.byte 0,12,0x04,0,0x80,6,6,0
1252	.long 0
1253	.size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1254	___
1255	}} }}}
1256
1257	#########################################################################
1258	{{{ # CTR procedure[s] #
1259	my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
1260	my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
1261	my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
1262	map("v$_",(4..11));
1263	my $dat=$tmp;
1264
1265	$code.=<<___;
1266	.globl .${prefix}_ctr32_encrypt_blocks
1267	.align 5
1268	.${prefix}_ctr32_encrypt_blocks:
1269	${UCMP}i $len,1
1270	bltlr-
1271
1272	lis r0,0xfff0
1273	mfspr $vrsave,256
1274	mtspr 256,r0
1275
1276	li $idx,15
1277	vxor $rndkey0,$rndkey0,$rndkey0
1278	le?vspltisb $tmp,0x0f
1279
1280	lvx $ivec,0,$ivp # load [unaligned] iv
1281	lvsl $inpperm,0,$ivp
1282	lvx $inptail,$idx,$ivp
1283	vspltisb $one,1
1284	le?vxor $inpperm,$inpperm,$tmp
1285	vperm $ivec,$ivec,$inptail,$inpperm
1286	vsldoi $one,$rndkey0,$one,1
1287
1288	neg r11,$inp
1289	?lvsl $keyperm,0,$key # prepare for unaligned key
1290	lwz $rounds,240($key)
1291
1292	lvsr $inpperm,0,r11 # prepare for unaligned load
1293	lvx $inptail,0,$inp
1294	addi $inp,$inp,15 # 15 is not typo
1295	le?vxor $inpperm,$inpperm,$tmp
1296
1297	srwi $rounds,$rounds,1
1298	li $idx,16
1299	subi $rounds,$rounds,1
1300
1301	${UCMP}i $len,8
1302	bge _aesp8_ctr32_encrypt8x
1303
1304	?lvsr $outperm,0,$out # prepare for unaligned store
1305	vspltisb $outmask,-1
1306	lvx $outhead,0,$out
1307	?vperm $outmask,$rndkey0,$outmask,$outperm
1308	le?vxor $outperm,$outperm,$tmp
1309
1310	lvx $rndkey0,0,$key
1311	mtctr $rounds
1312	lvx $rndkey1,$idx,$key
1313	addi $idx,$idx,16
1314	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1315	vxor $inout,$ivec,$rndkey0
1316	lvx $rndkey0,$idx,$key
1317	addi $idx,$idx,16
1318	b Loop_ctr32_enc
1319
1320	.align 5
1321	Loop_ctr32_enc:
1322	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1323	vcipher $inout,$inout,$rndkey1
1324	lvx $rndkey1,$idx,$key
1325	addi $idx,$idx,16
1326	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1327	vcipher $inout,$inout,$rndkey0
1328	lvx $rndkey0,$idx,$key
1329	addi $idx,$idx,16
1330	bdnz Loop_ctr32_enc
1331
1332	vadduwm $ivec,$ivec,$one
1333	vmr $dat,$inptail
1334	lvx $inptail,0,$inp
1335	addi $inp,$inp,16
1336	subic. $len,$len,1 # blocks--
1337
1338	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1339	vcipher $inout,$inout,$rndkey1
1340	lvx $rndkey1,$idx,$key
1341	vperm $dat,$dat,$inptail,$inpperm
1342	li $idx,16
1343	?vperm $rndkey1,$rndkey0,$rndkey1,$keyperm
1344	lvx $rndkey0,0,$key
1345	vxor $dat,$dat,$rndkey1 # last round key
1346	vcipherlast $inout,$inout,$dat
1347
1348	lvx $rndkey1,$idx,$key
1349	addi $idx,$idx,16
1350	vperm $inout,$inout,$inout,$outperm
1351	vsel $dat,$outhead,$inout,$outmask
1352	mtctr $rounds
1353	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1354	vmr $outhead,$inout
1355	vxor $inout,$ivec,$rndkey0
1356	lvx $rndkey0,$idx,$key
1357	addi $idx,$idx,16
1358	stvx $dat,0,$out
1359	addi $out,$out,16
1360	bne Loop_ctr32_enc
1361
1362	addi $out,$out,-1
1363	lvx $inout,0,$out # redundant in aligned case
1364	vsel $inout,$outhead,$inout,$outmask
1365	stvx $inout,0,$out
1366
1367	mtspr 256,$vrsave
1368	blr
1369	.long 0
1370	.byte 0,12,0x14,0,0,0,6,0
1371	.long 0
1372	___
1373	#########################################################################
1374	{{ # Optimized CTR procedure #
1375	my $key_="r11";
1376	my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
1377	$x00=0 if ($flavour =~ /osx/);
1378	my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1379	my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
1380	my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
1381	# v26-v31 last 6 round keys
1382	my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
1383	my ($two,$three,$four)=($outhead,$outperm,$outmask);
1384
1385	$code.=<<___;
1386	.align 5
1387	_aesp8_ctr32_encrypt8x:
1388	$STU $sp,-`($FRAME+2116+6$SIZE_T)`($sp)
1389	li r10,`$FRAME+8*16+15`
1390	li r11,`$FRAME+8*16+31`
1391	stvx v20,r10,$sp # ABI says so
1392	addi r10,r10,32
1393	stvx v21,r11,$sp
1394	addi r11,r11,32
1395	stvx v22,r10,$sp
1396	addi r10,r10,32
1397	stvx v23,r11,$sp
1398	addi r11,r11,32
1399	stvx v24,r10,$sp
1400	addi r10,r10,32
1401	stvx v25,r11,$sp
1402	addi r11,r11,32
1403	stvx v26,r10,$sp
1404	addi r10,r10,32
1405	stvx v27,r11,$sp
1406	addi r11,r11,32
1407	stvx v28,r10,$sp
1408	addi r10,r10,32
1409	stvx v29,r11,$sp
1410	addi r11,r11,32
1411	stvx v30,r10,$sp
1412	stvx v31,r11,$sp
1413	li r0,-1
1414	stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
1415	li $x10,0x10
1416	$PUSH r26,`$FRAME+2116+0$SIZE_T`($sp)
1417	li $x20,0x20
1418	$PUSH r27,`$FRAME+2116+1$SIZE_T`($sp)
1419	li $x30,0x30
1420	$PUSH r28,`$FRAME+2116+2$SIZE_T`($sp)
1421	li $x40,0x40
1422	$PUSH r29,`$FRAME+2116+3$SIZE_T`($sp)
1423	li $x50,0x50
1424	$PUSH r30,`$FRAME+2116+4$SIZE_T`($sp)
1425	li $x60,0x60
1426	$PUSH r31,`$FRAME+2116+5$SIZE_T`($sp)
1427	li $x70,0x70
1428	mtspr 256,r0
1429
1430	subi $rounds,$rounds,3 # -4 in total
1431
1432	lvx $rndkey0,$x00,$key # load key schedule
1433	lvx v30,$x10,$key
1434	addi $key,$key,0x20
1435	lvx v31,$x00,$key
1436	?vperm $rndkey0,$rndkey0,v30,$keyperm
1437	addi $key_,$sp,$FRAME+15
1438	mtctr $rounds
1439
1440	Load_ctr32_enc_key:
1441	?vperm v24,v30,v31,$keyperm
1442	lvx v30,$x10,$key
1443	addi $key,$key,0x20
1444	stvx v24,$x00,$key_ # off-load round[1]
1445	?vperm v25,v31,v30,$keyperm
1446	lvx v31,$x00,$key
1447	stvx v25,$x10,$key_ # off-load round[2]
1448	addi $key_,$key_,0x20
1449	bdnz Load_ctr32_enc_key
1450
1451	lvx v26,$x10,$key
1452	?vperm v24,v30,v31,$keyperm
1453	lvx v27,$x20,$key
1454	stvx v24,$x00,$key_ # off-load round[3]
1455	?vperm v25,v31,v26,$keyperm
1456	lvx v28,$x30,$key
1457	stvx v25,$x10,$key_ # off-load round[4]
1458	addi $key_,$sp,$FRAME+15 # rewind $key_
1459	?vperm v26,v26,v27,$keyperm
1460	lvx v29,$x40,$key
1461	?vperm v27,v27,v28,$keyperm
1462	lvx v30,$x50,$key
1463	?vperm v28,v28,v29,$keyperm
1464	lvx v31,$x60,$key
1465	?vperm v29,v29,v30,$keyperm
1466	lvx $out0,$x70,$key # borrow $out0
1467	?vperm v30,v30,v31,$keyperm
1468	lvx v24,$x00,$key_ # pre-load round[1]
1469	?vperm v31,v31,$out0,$keyperm
1470	lvx v25,$x10,$key_ # pre-load round[2]
1471
1472	vadduwm $two,$one,$one
1473	subi $inp,$inp,15 # undo "caller"
1474	$SHL $len,$len,4
1475
1476	vadduwm $out1,$ivec,$one # counter values ...
1477	vadduwm $out2,$ivec,$two
1478	vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1479	le?li $idx,8
1480	vadduwm $out3,$out1,$two
1481	vxor $out1,$out1,$rndkey0
1482	le?lvsl $inpperm,0,$idx
1483	vadduwm $out4,$out2,$two
1484	vxor $out2,$out2,$rndkey0
1485	le?vspltisb $tmp,0x0f
1486	vadduwm $out5,$out3,$two
1487	vxor $out3,$out3,$rndkey0
1488	le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
1489	vadduwm $out6,$out4,$two
1490	vxor $out4,$out4,$rndkey0
1491	vadduwm $out7,$out5,$two
1492	vxor $out5,$out5,$rndkey0
1493	vadduwm $ivec,$out6,$two # next counter value
1494	vxor $out6,$out6,$rndkey0
1495	vxor $out7,$out7,$rndkey0
1496
1497	mtctr $rounds
1498	b Loop_ctr32_enc8x
1499	.align 5
1500	Loop_ctr32_enc8x:
1501	vcipher $out0,$out0,v24
1502	vcipher $out1,$out1,v24
1503	vcipher $out2,$out2,v24
1504	vcipher $out3,$out3,v24
1505	vcipher $out4,$out4,v24
1506	vcipher $out5,$out5,v24
1507	vcipher $out6,$out6,v24
1508	vcipher $out7,$out7,v24
1509	Loop_ctr32_enc8x_middle:
1510	lvx v24,$x20,$key_ # round[3]
1511	addi $key_,$key_,0x20
1512
1513	vcipher $out0,$out0,v25
1514	vcipher $out1,$out1,v25
1515	vcipher $out2,$out2,v25
1516	vcipher $out3,$out3,v25
1517	vcipher $out4,$out4,v25
1518	vcipher $out5,$out5,v25
1519	vcipher $out6,$out6,v25
1520	vcipher $out7,$out7,v25
1521	lvx v25,$x10,$key_ # round[4]
1522	bdnz Loop_ctr32_enc8x
1523
1524	subic r11,$len,256 # $len-256, borrow $key_
1525	vcipher $out0,$out0,v24
1526	vcipher $out1,$out1,v24
1527	vcipher $out2,$out2,v24
1528	vcipher $out3,$out3,v24
1529	vcipher $out4,$out4,v24
1530	vcipher $out5,$out5,v24
1531	vcipher $out6,$out6,v24
1532	vcipher $out7,$out7,v24
1533
1534	subfe r0,r0,r0 # borrow?-1:0
1535	vcipher $out0,$out0,v25
1536	vcipher $out1,$out1,v25
1537	vcipher $out2,$out2,v25
1538	vcipher $out3,$out3,v25
1539	vcipher $out4,$out4,v25
1540	vcipher $out5,$out5,v25
1541	vcipher $out6,$out6,v25
1542	vcipher $out7,$out7,v25
1543
1544	and r0,r0,r11
1545	addi $key_,$sp,$FRAME+15 # rewind $key_
1546	vcipher $out0,$out0,v26
1547	vcipher $out1,$out1,v26
1548	vcipher $out2,$out2,v26
1549	vcipher $out3,$out3,v26
1550	vcipher $out4,$out4,v26
1551	vcipher $out5,$out5,v26
1552	vcipher $out6,$out6,v26
1553	vcipher $out7,$out7,v26
1554	lvx v24,$x00,$key_ # re-pre-load round[1]
1555
1556	subic $len,$len,129 # $len-=129
1557	vcipher $out0,$out0,v27
1558	addi $len,$len,1 # $len-=128 really
1559	vcipher $out1,$out1,v27
1560	vcipher $out2,$out2,v27
1561	vcipher $out3,$out3,v27
1562	vcipher $out4,$out4,v27
1563	vcipher $out5,$out5,v27
1564	vcipher $out6,$out6,v27
1565	vcipher $out7,$out7,v27
1566	lvx v25,$x10,$key_ # re-pre-load round[2]
1567
1568	vcipher $out0,$out0,v28
1569	lvx_u $in0,$x00,$inp # load input
1570	vcipher $out1,$out1,v28
1571	lvx_u $in1,$x10,$inp
1572	vcipher $out2,$out2,v28
1573	lvx_u $in2,$x20,$inp
1574	vcipher $out3,$out3,v28
1575	lvx_u $in3,$x30,$inp
1576	vcipher $out4,$out4,v28
1577	lvx_u $in4,$x40,$inp
1578	vcipher $out5,$out5,v28
1579	lvx_u $in5,$x50,$inp
1580	vcipher $out6,$out6,v28
1581	lvx_u $in6,$x60,$inp
1582	vcipher $out7,$out7,v28
1583	lvx_u $in7,$x70,$inp
1584	addi $inp,$inp,0x80
1585
1586	vcipher $out0,$out0,v29
1587	le?vperm $in0,$in0,$in0,$inpperm
1588	vcipher $out1,$out1,v29
1589	le?vperm $in1,$in1,$in1,$inpperm
1590	vcipher $out2,$out2,v29
1591	le?vperm $in2,$in2,$in2,$inpperm
1592	vcipher $out3,$out3,v29
1593	le?vperm $in3,$in3,$in3,$inpperm
1594	vcipher $out4,$out4,v29
1595	le?vperm $in4,$in4,$in4,$inpperm
1596	vcipher $out5,$out5,v29
1597	le?vperm $in5,$in5,$in5,$inpperm
1598	vcipher $out6,$out6,v29
1599	le?vperm $in6,$in6,$in6,$inpperm
1600	vcipher $out7,$out7,v29
1601	le?vperm $in7,$in7,$in7,$inpperm
1602
1603	add $inp,$inp,r0 # $inp is adjusted in such
1604	# way that at exit from the
1605	# loop inX-in7 are loaded
1606	# with last "words"
1607	subfe. r0,r0,r0 # borrow?-1:0
1608	vcipher $out0,$out0,v30
1609	vxor $in0,$in0,v31 # xor with last round key
1610	vcipher $out1,$out1,v30
1611	vxor $in1,$in1,v31
1612	vcipher $out2,$out2,v30
1613	vxor $in2,$in2,v31
1614	vcipher $out3,$out3,v30
1615	vxor $in3,$in3,v31
1616	vcipher $out4,$out4,v30
1617	vxor $in4,$in4,v31
1618	vcipher $out5,$out5,v30
1619	vxor $in5,$in5,v31
1620	vcipher $out6,$out6,v30
1621	vxor $in6,$in6,v31
1622	vcipher $out7,$out7,v30
1623	vxor $in7,$in7,v31
1624
1625	bne Lctr32_enc8x_break # did $len-129 borrow?
1626
1627	vcipherlast $in0,$out0,$in0
1628	vcipherlast $in1,$out1,$in1
1629	vadduwm $out1,$ivec,$one # counter values ...
1630	vcipherlast $in2,$out2,$in2
1631	vadduwm $out2,$ivec,$two
1632	vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1633	vcipherlast $in3,$out3,$in3
1634	vadduwm $out3,$out1,$two
1635	vxor $out1,$out1,$rndkey0
1636	vcipherlast $in4,$out4,$in4
1637	vadduwm $out4,$out2,$two
1638	vxor $out2,$out2,$rndkey0
1639	vcipherlast $in5,$out5,$in5
1640	vadduwm $out5,$out3,$two
1641	vxor $out3,$out3,$rndkey0
1642	vcipherlast $in6,$out6,$in6
1643	vadduwm $out6,$out4,$two
1644	vxor $out4,$out4,$rndkey0
1645	vcipherlast $in7,$out7,$in7
1646	vadduwm $out7,$out5,$two
1647	vxor $out5,$out5,$rndkey0
1648	le?vperm $in0,$in0,$in0,$inpperm
1649	vadduwm $ivec,$out6,$two # next counter value
1650	vxor $out6,$out6,$rndkey0
1651	le?vperm $in1,$in1,$in1,$inpperm
1652	vxor $out7,$out7,$rndkey0
1653	mtctr $rounds
1654
1655	vcipher $out0,$out0,v24
1656	stvx_u $in0,$x00,$out
1657	le?vperm $in2,$in2,$in2,$inpperm
1658	vcipher $out1,$out1,v24
1659	stvx_u $in1,$x10,$out
1660	le?vperm $in3,$in3,$in3,$inpperm
1661	vcipher $out2,$out2,v24
1662	stvx_u $in2,$x20,$out
1663	le?vperm $in4,$in4,$in4,$inpperm
1664	vcipher $out3,$out3,v24
1665	stvx_u $in3,$x30,$out
1666	le?vperm $in5,$in5,$in5,$inpperm
1667	vcipher $out4,$out4,v24
1668	stvx_u $in4,$x40,$out
1669	le?vperm $in6,$in6,$in6,$inpperm
1670	vcipher $out5,$out5,v24
1671	stvx_u $in5,$x50,$out
1672	le?vperm $in7,$in7,$in7,$inpperm
1673	vcipher $out6,$out6,v24
1674	stvx_u $in6,$x60,$out
1675	vcipher $out7,$out7,v24
1676	stvx_u $in7,$x70,$out
1677	addi $out,$out,0x80
1678
1679	b Loop_ctr32_enc8x_middle
1680
1681	.align 5
1682	Lctr32_enc8x_break:
1683	cmpwi $len,-0x60
1684	blt Lctr32_enc8x_one
1685	nop
1686	beq Lctr32_enc8x_two
1687	cmpwi $len,-0x40
1688	blt Lctr32_enc8x_three
1689	nop
1690	beq Lctr32_enc8x_four
1691	cmpwi $len,-0x20
1692	blt Lctr32_enc8x_five
1693	nop
1694	beq Lctr32_enc8x_six
1695	cmpwi $len,0x00
1696	blt Lctr32_enc8x_seven
1697
1698	Lctr32_enc8x_eight:
1699	vcipherlast $out0,$out0,$in0
1700	vcipherlast $out1,$out1,$in1
1701	vcipherlast $out2,$out2,$in2
1702	vcipherlast $out3,$out3,$in3
1703	vcipherlast $out4,$out4,$in4
1704	vcipherlast $out5,$out5,$in5
1705	vcipherlast $out6,$out6,$in6
1706	vcipherlast $out7,$out7,$in7
1707
1708	le?vperm $out0,$out0,$out0,$inpperm
1709	le?vperm $out1,$out1,$out1,$inpperm
1710	stvx_u $out0,$x00,$out
1711	le?vperm $out2,$out2,$out2,$inpperm
1712	stvx_u $out1,$x10,$out
1713	le?vperm $out3,$out3,$out3,$inpperm
1714	stvx_u $out2,$x20,$out
1715	le?vperm $out4,$out4,$out4,$inpperm
1716	stvx_u $out3,$x30,$out
1717	le?vperm $out5,$out5,$out5,$inpperm
1718	stvx_u $out4,$x40,$out
1719	le?vperm $out6,$out6,$out6,$inpperm
1720	stvx_u $out5,$x50,$out
1721	le?vperm $out7,$out7,$out7,$inpperm
1722	stvx_u $out6,$x60,$out
1723	stvx_u $out7,$x70,$out
1724	addi $out,$out,0x80
1725	b Lctr32_enc8x_done
1726
1727	.align 5
1728	Lctr32_enc8x_seven:
1729	vcipherlast $out0,$out0,$in1
1730	vcipherlast $out1,$out1,$in2
1731	vcipherlast $out2,$out2,$in3
1732	vcipherlast $out3,$out3,$in4
1733	vcipherlast $out4,$out4,$in5
1734	vcipherlast $out5,$out5,$in6
1735	vcipherlast $out6,$out6,$in7
1736
1737	le?vperm $out0,$out0,$out0,$inpperm
1738	le?vperm $out1,$out1,$out1,$inpperm
1739	stvx_u $out0,$x00,$out
1740	le?vperm $out2,$out2,$out2,$inpperm
1741	stvx_u $out1,$x10,$out
1742	le?vperm $out3,$out3,$out3,$inpperm
1743	stvx_u $out2,$x20,$out
1744	le?vperm $out4,$out4,$out4,$inpperm
1745	stvx_u $out3,$x30,$out
1746	le?vperm $out5,$out5,$out5,$inpperm
1747	stvx_u $out4,$x40,$out
1748	le?vperm $out6,$out6,$out6,$inpperm
1749	stvx_u $out5,$x50,$out
1750	stvx_u $out6,$x60,$out
1751	addi $out,$out,0x70
1752	b Lctr32_enc8x_done
1753
1754	.align 5
1755	Lctr32_enc8x_six:
1756	vcipherlast $out0,$out0,$in2
1757	vcipherlast $out1,$out1,$in3
1758	vcipherlast $out2,$out2,$in4
1759	vcipherlast $out3,$out3,$in5
1760	vcipherlast $out4,$out4,$in6
1761	vcipherlast $out5,$out5,$in7
1762
1763	le?vperm $out0,$out0,$out0,$inpperm
1764	le?vperm $out1,$out1,$out1,$inpperm
1765	stvx_u $out0,$x00,$out
1766	le?vperm $out2,$out2,$out2,$inpperm
1767	stvx_u $out1,$x10,$out
1768	le?vperm $out3,$out3,$out3,$inpperm
1769	stvx_u $out2,$x20,$out
1770	le?vperm $out4,$out4,$out4,$inpperm
1771	stvx_u $out3,$x30,$out
1772	le?vperm $out5,$out5,$out5,$inpperm
1773	stvx_u $out4,$x40,$out
1774	stvx_u $out5,$x50,$out
1775	addi $out,$out,0x60
1776	b Lctr32_enc8x_done
1777
1778	.align 5
1779	Lctr32_enc8x_five:
1780	vcipherlast $out0,$out0,$in3
1781	vcipherlast $out1,$out1,$in4
1782	vcipherlast $out2,$out2,$in5
1783	vcipherlast $out3,$out3,$in6
1784	vcipherlast $out4,$out4,$in7
1785
1786	le?vperm $out0,$out0,$out0,$inpperm
1787	le?vperm $out1,$out1,$out1,$inpperm
1788	stvx_u $out0,$x00,$out
1789	le?vperm $out2,$out2,$out2,$inpperm
1790	stvx_u $out1,$x10,$out
1791	le?vperm $out3,$out3,$out3,$inpperm
1792	stvx_u $out2,$x20,$out
1793	le?vperm $out4,$out4,$out4,$inpperm
1794	stvx_u $out3,$x30,$out
1795	stvx_u $out4,$x40,$out
1796	addi $out,$out,0x50
1797	b Lctr32_enc8x_done
1798
1799	.align 5
1800	Lctr32_enc8x_four:
1801	vcipherlast $out0,$out0,$in4
1802	vcipherlast $out1,$out1,$in5
1803	vcipherlast $out2,$out2,$in6
1804	vcipherlast $out3,$out3,$in7
1805
1806	le?vperm $out0,$out0,$out0,$inpperm
1807	le?vperm $out1,$out1,$out1,$inpperm
1808	stvx_u $out0,$x00,$out
1809	le?vperm $out2,$out2,$out2,$inpperm
1810	stvx_u $out1,$x10,$out
1811	le?vperm $out3,$out3,$out3,$inpperm
1812	stvx_u $out2,$x20,$out
1813	stvx_u $out3,$x30,$out
1814	addi $out,$out,0x40
1815	b Lctr32_enc8x_done
1816
1817	.align 5
1818	Lctr32_enc8x_three:
1819	vcipherlast $out0,$out0,$in5
1820	vcipherlast $out1,$out1,$in6
1821	vcipherlast $out2,$out2,$in7
1822
1823	le?vperm $out0,$out0,$out0,$inpperm
1824	le?vperm $out1,$out1,$out1,$inpperm
1825	stvx_u $out0,$x00,$out
1826	le?vperm $out2,$out2,$out2,$inpperm
1827	stvx_u $out1,$x10,$out
1828	stvx_u $out2,$x20,$out
1829	addi $out,$out,0x30
1830	b Lcbc_dec8x_done
1831
1832	.align 5
1833	Lctr32_enc8x_two:
1834	vcipherlast $out0,$out0,$in6
1835	vcipherlast $out1,$out1,$in7
1836
1837	le?vperm $out0,$out0,$out0,$inpperm
1838	le?vperm $out1,$out1,$out1,$inpperm
1839	stvx_u $out0,$x00,$out
1840	stvx_u $out1,$x10,$out
1841	addi $out,$out,0x20
1842	b Lcbc_dec8x_done
1843
1844	.align 5
1845	Lctr32_enc8x_one:
1846	vcipherlast $out0,$out0,$in7
1847
1848	le?vperm $out0,$out0,$out0,$inpperm
1849	stvx_u $out0,0,$out
1850	addi $out,$out,0x10
1851
1852	Lctr32_enc8x_done:
1853	li r10,`$FRAME+15`
1854	li r11,`$FRAME+31`
1855	stvx $inpperm,r10,$sp # wipe copies of round keys
1856	addi r10,r10,32
1857	stvx $inpperm,r11,$sp
1858	addi r11,r11,32
1859	stvx $inpperm,r10,$sp
1860	addi r10,r10,32
1861	stvx $inpperm,r11,$sp
1862	addi r11,r11,32
1863	stvx $inpperm,r10,$sp
1864	addi r10,r10,32
1865	stvx $inpperm,r11,$sp
1866	addi r11,r11,32
1867	stvx $inpperm,r10,$sp
1868	addi r10,r10,32
1869	stvx $inpperm,r11,$sp
1870	addi r11,r11,32
1871
1872	mtspr 256,$vrsave
1873	lvx v20,r10,$sp # ABI says so
1874	addi r10,r10,32
1875	lvx v21,r11,$sp
1876	addi r11,r11,32
1877	lvx v22,r10,$sp
1878	addi r10,r10,32
1879	lvx v23,r11,$sp
1880	addi r11,r11,32
1881	lvx v24,r10,$sp
1882	addi r10,r10,32
1883	lvx v25,r11,$sp
1884	addi r11,r11,32
1885	lvx v26,r10,$sp
1886	addi r10,r10,32
1887	lvx v27,r11,$sp
1888	addi r11,r11,32
1889	lvx v28,r10,$sp
1890	addi r10,r10,32
1891	lvx v29,r11,$sp
1892	addi r11,r11,32
1893	lvx v30,r10,$sp
1894	lvx v31,r11,$sp
1895	$POP r26,`$FRAME+2116+0$SIZE_T`($sp)
1896	$POP r27,`$FRAME+2116+1$SIZE_T`($sp)
1897	$POP r28,`$FRAME+2116+2$SIZE_T`($sp)
1898	$POP r29,`$FRAME+2116+3$SIZE_T`($sp)
1899	$POP r30,`$FRAME+2116+4$SIZE_T`($sp)
1900	$POP r31,`$FRAME+2116+5$SIZE_T`($sp)
1901	addi $sp,$sp,`$FRAME+2116+6$SIZE_T`
1902	blr
1903	.long 0
1904	.byte 0,12,0x04,0,0x80,6,6,0
1905	.long 0
1906	.size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1907	___
1908	}} }}}
1909
1910	#########################################################################
1911	{{{ # XTS procedures #
1912	# int aes_p8_xts_[en\|de]crypt(const char inp, char out, size_t len, #
1913	# const AES_KEY key1, const AES_KEY key2, #
1914	# [const] unsigned char iv[16]); #
1915	# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which #
1916	# input tweak value is assumed to be encrypted already, and last tweak #
1917	# value, one suitable for consecutive call on same chunk of data, is #
1918	# written back to original buffer. In addition, in "tweak chaining" #
1919	# mode only complete input blocks are processed. #
1920
1921	my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10));
1922	my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2));
1923	my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7));
1924	my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12));
1925	my $taillen = $key2;
1926
1927	($inp,$idx) = ($idx,$inp); # reassign
1928
1929	$code.=<<___;
1930	.globl .${prefix}_xts_encrypt
1931	.align 5
1932	.${prefix}_xts_encrypt:
1933	mr $inp,r3 # reassign
1934	li r3,-1
1935	${UCMP}i $len,16
1936	bltlr-
1937
1938	lis r0,0xfff0
1939	mfspr r12,256 # save vrsave
1940	li r11,0
1941	mtspr 256,r0
1942
1943	vspltisb $seven,0x07 # 0x070707..07
1944	le?lvsl $leperm,r11,r11
1945	le?vspltisb $tmp,0x0f
1946	le?vxor $leperm,$leperm,$seven
1947
1948	li $idx,15
1949	lvx $tweak,0,$ivp # load [unaligned] iv
1950	lvsl $inpperm,0,$ivp
1951	lvx $inptail,$idx,$ivp
1952	le?vxor $inpperm,$inpperm,$tmp
1953	vperm $tweak,$tweak,$inptail,$inpperm
1954
1955	neg r11,$inp
1956	lvsr $inpperm,0,r11 # prepare for unaligned load
1957	lvx $inout,0,$inp
1958	addi $inp,$inp,15 # 15 is not typo
1959	le?vxor $inpperm,$inpperm,$tmp
1960
1961	${UCMP}i $key2,0 # key2==NULL?
1962	beq Lxts_enc_no_key2
1963
1964	?lvsl $keyperm,0,$key2 # prepare for unaligned key
1965	lwz $rounds,240($key2)
1966	srwi $rounds,$rounds,1
1967	subi $rounds,$rounds,1
1968	li $idx,16
1969
1970	lvx $rndkey0,0,$key2
1971	lvx $rndkey1,$idx,$key2
1972	addi $idx,$idx,16
1973	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1974	vxor $tweak,$tweak,$rndkey0
1975	lvx $rndkey0,$idx,$key2
1976	addi $idx,$idx,16
1977	mtctr $rounds
1978
1979	Ltweak_xts_enc:
1980	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1981	vcipher $tweak,$tweak,$rndkey1
1982	lvx $rndkey1,$idx,$key2
1983	addi $idx,$idx,16
1984	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1985	vcipher $tweak,$tweak,$rndkey0
1986	lvx $rndkey0,$idx,$key2
1987	addi $idx,$idx,16
1988	bdnz Ltweak_xts_enc
1989
1990	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1991	vcipher $tweak,$tweak,$rndkey1
1992	lvx $rndkey1,$idx,$key2
1993	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1994	vcipherlast $tweak,$tweak,$rndkey0
1995
1996	li $ivp,0 # don't chain the tweak
1997	b Lxts_enc
1998
1999	Lxts_enc_no_key2:
2000	li $idx,-16
2001	and $len,$len,$idx # in "tweak chaining"
2002	# mode only complete
2003	# blocks are processed
2004	Lxts_enc:
2005	lvx $inptail,0,$inp
2006	addi $inp,$inp,16
2007
2008	?lvsl $keyperm,0,$key1 # prepare for unaligned key
2009	lwz $rounds,240($key1)
2010	srwi $rounds,$rounds,1
2011	subi $rounds,$rounds,1
2012	li $idx,16
2013
2014	vslb $eighty7,$seven,$seven # 0x808080..80
2015	vor $eighty7,$eighty7,$seven # 0x878787..87
2016	vspltisb $tmp,1 # 0x010101..01
2017	vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
2018
2019	${UCMP}i $len,96
2020	bge _aesp8_xts_encrypt6x
2021
2022	andi. $taillen,$len,15
2023	subic r0,$len,32
2024	subi $taillen,$taillen,16
2025	subfe r0,r0,r0
2026	and r0,r0,$taillen
2027	add $inp,$inp,r0
2028
2029	lvx $rndkey0,0,$key1
2030	lvx $rndkey1,$idx,$key1
2031	addi $idx,$idx,16
2032	vperm $inout,$inout,$inptail,$inpperm
2033	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2034	vxor $inout,$inout,$tweak
2035	vxor $inout,$inout,$rndkey0
2036	lvx $rndkey0,$idx,$key1
2037	addi $idx,$idx,16
2038	mtctr $rounds
2039	b Loop_xts_enc
2040
2041	.align 5
2042	Loop_xts_enc:
2043	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2044	vcipher $inout,$inout,$rndkey1
2045	lvx $rndkey1,$idx,$key1
2046	addi $idx,$idx,16
2047	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2048	vcipher $inout,$inout,$rndkey0
2049	lvx $rndkey0,$idx,$key1
2050	addi $idx,$idx,16
2051	bdnz Loop_xts_enc
2052
2053	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2054	vcipher $inout,$inout,$rndkey1
2055	lvx $rndkey1,$idx,$key1
2056	li $idx,16
2057	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2058	vxor $rndkey0,$rndkey0,$tweak
2059	vcipherlast $output,$inout,$rndkey0
2060
2061	le?vperm $tmp,$output,$output,$leperm
2062	be?nop
2063	le?stvx_u $tmp,0,$out
2064	be?stvx_u $output,0,$out
2065	addi $out,$out,16
2066
2067	subic. $len,$len,16
2068	beq Lxts_enc_done
2069
2070	vmr $inout,$inptail
2071	lvx $inptail,0,$inp
2072	addi $inp,$inp,16
2073	lvx $rndkey0,0,$key1
2074	lvx $rndkey1,$idx,$key1
2075	addi $idx,$idx,16
2076
2077	subic r0,$len,32
2078	subfe r0,r0,r0
2079	and r0,r0,$taillen
2080	add $inp,$inp,r0
2081
2082	vsrab $tmp,$tweak,$seven # next tweak value
2083	vaddubm $tweak,$tweak,$tweak
2084	vsldoi $tmp,$tmp,$tmp,15
2085	vand $tmp,$tmp,$eighty7
2086	vxor $tweak,$tweak,$tmp
2087
2088	vperm $inout,$inout,$inptail,$inpperm
2089	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2090	vxor $inout,$inout,$tweak
2091	vxor $output,$output,$rndkey0 # just in case $len<16
2092	vxor $inout,$inout,$rndkey0
2093	lvx $rndkey0,$idx,$key1
2094	addi $idx,$idx,16
2095
2096	mtctr $rounds
2097	${UCMP}i $len,16
2098	bge Loop_xts_enc
2099
2100	vxor $output,$output,$tweak
2101	lvsr $inpperm,0,$len # $inpperm is no longer needed
2102	vxor $inptail,$inptail,$inptail # $inptail is no longer needed
2103	vspltisb $tmp,-1
2104	vperm $inptail,$inptail,$tmp,$inpperm
2105	vsel $inout,$inout,$output,$inptail
2106
2107	subi r11,$out,17
2108	subi $out,$out,16
2109	mtctr $len
2110	li $len,16
2111	Loop_xts_enc_steal:
2112	lbzu r0,1(r11)
2113	stb r0,16(r11)
2114	bdnz Loop_xts_enc_steal
2115
2116	mtctr $rounds
2117	b Loop_xts_enc # one more time...
2118
2119	Lxts_enc_done:
2120	${UCMP}i $ivp,0
2121	beq Lxts_enc_ret
2122
2123	vsrab $tmp,$tweak,$seven # next tweak value
2124	vaddubm $tweak,$tweak,$tweak
2125	vsldoi $tmp,$tmp,$tmp,15
2126	vand $tmp,$tmp,$eighty7
2127	vxor $tweak,$tweak,$tmp
2128
2129	le?vperm $tweak,$tweak,$tweak,$leperm
2130	stvx_u $tweak,0,$ivp
2131
2132	Lxts_enc_ret:
2133	mtspr 256,r12 # restore vrsave
2134	li r3,0
2135	blr
2136	.long 0
2137	.byte 0,12,0x04,0,0x80,6,6,0
2138	.long 0
2139	.size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2140
2141	.globl .${prefix}_xts_decrypt
2142	.align 5
2143	.${prefix}_xts_decrypt:
2144	mr $inp,r3 # reassign
2145	li r3,-1
2146	${UCMP}i $len,16
2147	bltlr-
2148
2149	lis r0,0xfff8
2150	mfspr r12,256 # save vrsave
2151	li r11,0
2152	mtspr 256,r0
2153
2154	andi. r0,$len,15
2155	neg r0,r0
2156	andi. r0,r0,16
2157	sub $len,$len,r0
2158
2159	vspltisb $seven,0x07 # 0x070707..07
2160	le?lvsl $leperm,r11,r11
2161	le?vspltisb $tmp,0x0f
2162	le?vxor $leperm,$leperm,$seven
2163
2164	li $idx,15
2165	lvx $tweak,0,$ivp # load [unaligned] iv
2166	lvsl $inpperm,0,$ivp
2167	lvx $inptail,$idx,$ivp
2168	le?vxor $inpperm,$inpperm,$tmp
2169	vperm $tweak,$tweak,$inptail,$inpperm
2170
2171	neg r11,$inp
2172	lvsr $inpperm,0,r11 # prepare for unaligned load
2173	lvx $inout,0,$inp
2174	addi $inp,$inp,15 # 15 is not typo
2175	le?vxor $inpperm,$inpperm,$tmp
2176
2177	${UCMP}i $key2,0 # key2==NULL?
2178	beq Lxts_dec_no_key2
2179
2180	?lvsl $keyperm,0,$key2 # prepare for unaligned key
2181	lwz $rounds,240($key2)
2182	srwi $rounds,$rounds,1
2183	subi $rounds,$rounds,1
2184	li $idx,16
2185
2186	lvx $rndkey0,0,$key2
2187	lvx $rndkey1,$idx,$key2
2188	addi $idx,$idx,16
2189	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2190	vxor $tweak,$tweak,$rndkey0
2191	lvx $rndkey0,$idx,$key2
2192	addi $idx,$idx,16
2193	mtctr $rounds
2194
2195	Ltweak_xts_dec:
2196	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2197	vcipher $tweak,$tweak,$rndkey1
2198	lvx $rndkey1,$idx,$key2
2199	addi $idx,$idx,16
2200	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2201	vcipher $tweak,$tweak,$rndkey0
2202	lvx $rndkey0,$idx,$key2
2203	addi $idx,$idx,16
2204	bdnz Ltweak_xts_dec
2205
2206	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2207	vcipher $tweak,$tweak,$rndkey1
2208	lvx $rndkey1,$idx,$key2
2209	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2210	vcipherlast $tweak,$tweak,$rndkey0
2211
2212	li $ivp,0 # don't chain the tweak
2213	b Lxts_dec
2214
2215	Lxts_dec_no_key2:
2216	neg $idx,$len
2217	andi. $idx,$idx,15
2218	add $len,$len,$idx # in "tweak chaining"
2219	# mode only complete
2220	# blocks are processed
2221	Lxts_dec:
2222	lvx $inptail,0,$inp
2223	addi $inp,$inp,16
2224
2225	?lvsl $keyperm,0,$key1 # prepare for unaligned key
2226	lwz $rounds,240($key1)
2227	srwi $rounds,$rounds,1
2228	subi $rounds,$rounds,1
2229	li $idx,16
2230
2231	vslb $eighty7,$seven,$seven # 0x808080..80
2232	vor $eighty7,$eighty7,$seven # 0x878787..87
2233	vspltisb $tmp,1 # 0x010101..01
2234	vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
2235
2236	${UCMP}i $len,96
2237	bge _aesp8_xts_decrypt6x
2238
2239	lvx $rndkey0,0,$key1
2240	lvx $rndkey1,$idx,$key1
2241	addi $idx,$idx,16
2242	vperm $inout,$inout,$inptail,$inpperm
2243	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2244	vxor $inout,$inout,$tweak
2245	vxor $inout,$inout,$rndkey0
2246	lvx $rndkey0,$idx,$key1
2247	addi $idx,$idx,16
2248	mtctr $rounds
2249
2250	${UCMP}i $len,16
2251	blt Ltail_xts_dec
2252	be?b Loop_xts_dec
2253
2254	.align 5
2255	Loop_xts_dec:
2256	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2257	vncipher $inout,$inout,$rndkey1
2258	lvx $rndkey1,$idx,$key1
2259	addi $idx,$idx,16
2260	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2261	vncipher $inout,$inout,$rndkey0
2262	lvx $rndkey0,$idx,$key1
2263	addi $idx,$idx,16
2264	bdnz Loop_xts_dec
2265
2266	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2267	vncipher $inout,$inout,$rndkey1
2268	lvx $rndkey1,$idx,$key1
2269	li $idx,16
2270	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2271	vxor $rndkey0,$rndkey0,$tweak
2272	vncipherlast $output,$inout,$rndkey0
2273
2274	le?vperm $tmp,$output,$output,$leperm
2275	be?nop
2276	le?stvx_u $tmp,0,$out
2277	be?stvx_u $output,0,$out
2278	addi $out,$out,16
2279
2280	subic. $len,$len,16
2281	beq Lxts_dec_done
2282
2283	vmr $inout,$inptail
2284	lvx $inptail,0,$inp
2285	addi $inp,$inp,16
2286	lvx $rndkey0,0,$key1
2287	lvx $rndkey1,$idx,$key1
2288	addi $idx,$idx,16
2289
2290	vsrab $tmp,$tweak,$seven # next tweak value
2291	vaddubm $tweak,$tweak,$tweak
2292	vsldoi $tmp,$tmp,$tmp,15
2293	vand $tmp,$tmp,$eighty7
2294	vxor $tweak,$tweak,$tmp
2295
2296	vperm $inout,$inout,$inptail,$inpperm
2297	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2298	vxor $inout,$inout,$tweak
2299	vxor $inout,$inout,$rndkey0
2300	lvx $rndkey0,$idx,$key1
2301	addi $idx,$idx,16
2302
2303	mtctr $rounds
2304	${UCMP}i $len,16
2305	bge Loop_xts_dec
2306
2307	Ltail_xts_dec:
2308	vsrab $tmp,$tweak,$seven # next tweak value
2309	vaddubm $tweak1,$tweak,$tweak
2310	vsldoi $tmp,$tmp,$tmp,15
2311	vand $tmp,$tmp,$eighty7
2312	vxor $tweak1,$tweak1,$tmp
2313
2314	subi $inp,$inp,16
2315	add $inp,$inp,$len
2316
2317	vxor $inout,$inout,$tweak # :-(
2318	vxor $inout,$inout,$tweak1 # :-)
2319
2320	Loop_xts_dec_short:
2321	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2322	vncipher $inout,$inout,$rndkey1
2323	lvx $rndkey1,$idx,$key1
2324	addi $idx,$idx,16
2325	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2326	vncipher $inout,$inout,$rndkey0
2327	lvx $rndkey0,$idx,$key1
2328	addi $idx,$idx,16
2329	bdnz Loop_xts_dec_short
2330
2331	?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2332	vncipher $inout,$inout,$rndkey1
2333	lvx $rndkey1,$idx,$key1
2334	li $idx,16
2335	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2336	vxor $rndkey0,$rndkey0,$tweak1
2337	vncipherlast $output,$inout,$rndkey0
2338
2339	le?vperm $tmp,$output,$output,$leperm
2340	be?nop
2341	le?stvx_u $tmp,0,$out
2342	be?stvx_u $output,0,$out
2343
2344	vmr $inout,$inptail
2345	lvx $inptail,0,$inp
2346	#addi $inp,$inp,16
2347	lvx $rndkey0,0,$key1
2348	lvx $rndkey1,$idx,$key1
2349	addi $idx,$idx,16
2350	vperm $inout,$inout,$inptail,$inpperm
2351	?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2352
2353	lvsr $inpperm,0,$len # $inpperm is no longer needed
2354	vxor $inptail,$inptail,$inptail # $inptail is no longer needed
2355	vspltisb $tmp,-1
2356	vperm $inptail,$inptail,$tmp,$inpperm
2357	vsel $inout,$inout,$output,$inptail
2358
2359	vxor $rndkey0,$rndkey0,$tweak
2360	vxor $inout,$inout,$rndkey0
2361	lvx $rndkey0,$idx,$key1
2362	addi $idx,$idx,16
2363
2364	subi r11,$out,1
2365	mtctr $len
2366	li $len,16
2367	Loop_xts_dec_steal:
2368	lbzu r0,1(r11)
2369	stb r0,16(r11)
2370	bdnz Loop_xts_dec_steal
2371
2372	mtctr $rounds
2373	b Loop_xts_dec # one more time...
2374
2375	Lxts_dec_done:
2376	${UCMP}i $ivp,0
2377	beq Lxts_dec_ret
2378
2379	vsrab $tmp,$tweak,$seven # next tweak value
2380	vaddubm $tweak,$tweak,$tweak
2381	vsldoi $tmp,$tmp,$tmp,15
2382	vand $tmp,$tmp,$eighty7
2383	vxor $tweak,$tweak,$tmp
2384
2385	le?vperm $tweak,$tweak,$tweak,$leperm
2386	stvx_u $tweak,0,$ivp
2387
2388	Lxts_dec_ret:
2389	mtspr 256,r12 # restore vrsave
2390	li r3,0
2391	blr
2392	.long 0
2393	.byte 0,12,0x04,0,0x80,6,6,0
2394	.long 0
2395	.size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2396	___
2397	#########################################################################
2398	{{ # Optimized XTS procedures #
2399	my $key_=$key2;
2400	my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
2401	$x00=0 if ($flavour =~ /osx/);
2402	my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5));
2403	my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
2404	my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
2405	my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
2406	# v26-v31 last 6 round keys
2407	my ($keyperm)=($out0); # aliases with "caller", redundant assignment
2408	my $taillen=$x70;
2409
2410	$code.=<<___;
2411	.align 5
2412	_aesp8_xts_encrypt6x:
2413	$STU $sp,-`($FRAME+2116+6$SIZE_T)`($sp)
2414	mflr r11
2415	li r7,`$FRAME+8*16+15`
2416	li r3,`$FRAME+8*16+31`
2417	$PUSH r11,`$FRAME+2116+6$SIZE_T+$LRSAVE`($sp)
2418	stvx v20,r7,$sp # ABI says so
2419	addi r7,r7,32
2420	stvx v21,r3,$sp
2421	addi r3,r3,32
2422	stvx v22,r7,$sp
2423	addi r7,r7,32
2424	stvx v23,r3,$sp
2425	addi r3,r3,32
2426	stvx v24,r7,$sp
2427	addi r7,r7,32
2428	stvx v25,r3,$sp
2429	addi r3,r3,32
2430	stvx v26,r7,$sp
2431	addi r7,r7,32
2432	stvx v27,r3,$sp
2433	addi r3,r3,32
2434	stvx v28,r7,$sp
2435	addi r7,r7,32
2436	stvx v29,r3,$sp
2437	addi r3,r3,32
2438	stvx v30,r7,$sp
2439	stvx v31,r3,$sp
2440	li r0,-1
2441	stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
2442	li $x10,0x10
2443	$PUSH r26,`$FRAME+2116+0$SIZE_T`($sp)
2444	li $x20,0x20
2445	$PUSH r27,`$FRAME+2116+1$SIZE_T`($sp)
2446	li $x30,0x30
2447	$PUSH r28,`$FRAME+2116+2$SIZE_T`($sp)
2448	li $x40,0x40
2449	$PUSH r29,`$FRAME+2116+3$SIZE_T`($sp)
2450	li $x50,0x50
2451	$PUSH r30,`$FRAME+2116+4$SIZE_T`($sp)
2452	li $x60,0x60
2453	$PUSH r31,`$FRAME+2116+5$SIZE_T`($sp)
2454	li $x70,0x70
2455	mtspr 256,r0
2456
2457	subi $rounds,$rounds,3 # -4 in total
2458
2459	lvx $rndkey0,$x00,$key1 # load key schedule
2460	lvx v30,$x10,$key1
2461	addi $key1,$key1,0x20
2462	lvx v31,$x00,$key1
2463	?vperm $rndkey0,$rndkey0,v30,$keyperm
2464	addi $key_,$sp,$FRAME+15
2465	mtctr $rounds
2466
2467	Load_xts_enc_key:
2468	?vperm v24,v30,v31,$keyperm
2469	lvx v30,$x10,$key1
2470	addi $key1,$key1,0x20
2471	stvx v24,$x00,$key_ # off-load round[1]
2472	?vperm v25,v31,v30,$keyperm
2473	lvx v31,$x00,$key1
2474	stvx v25,$x10,$key_ # off-load round[2]
2475	addi $key_,$key_,0x20
2476	bdnz Load_xts_enc_key
2477
2478	lvx v26,$x10,$key1
2479	?vperm v24,v30,v31,$keyperm
2480	lvx v27,$x20,$key1
2481	stvx v24,$x00,$key_ # off-load round[3]
2482	?vperm v25,v31,v26,$keyperm
2483	lvx v28,$x30,$key1
2484	stvx v25,$x10,$key_ # off-load round[4]
2485	addi $key_,$sp,$FRAME+15 # rewind $key_
2486	?vperm v26,v26,v27,$keyperm
2487	lvx v29,$x40,$key1
2488	?vperm v27,v27,v28,$keyperm
2489	lvx v30,$x50,$key1
2490	?vperm v28,v28,v29,$keyperm
2491	lvx v31,$x60,$key1
2492	?vperm v29,v29,v30,$keyperm
2493	lvx $twk5,$x70,$key1 # borrow $twk5
2494	?vperm v30,v30,v31,$keyperm
2495	lvx v24,$x00,$key_ # pre-load round[1]
2496	?vperm v31,v31,$twk5,$keyperm
2497	lvx v25,$x10,$key_ # pre-load round[2]
2498
2499	vperm $in0,$inout,$inptail,$inpperm
2500	subi $inp,$inp,31 # undo "caller"
2501	vxor $twk0,$tweak,$rndkey0
2502	vsrab $tmp,$tweak,$seven # next tweak value
2503	vaddubm $tweak,$tweak,$tweak
2504	vsldoi $tmp,$tmp,$tmp,15
2505	vand $tmp,$tmp,$eighty7
2506	vxor $out0,$in0,$twk0
2507	vxor $tweak,$tweak,$tmp
2508
2509	lvx_u $in1,$x10,$inp
2510	vxor $twk1,$tweak,$rndkey0
2511	vsrab $tmp,$tweak,$seven # next tweak value
2512	vaddubm $tweak,$tweak,$tweak
2513	vsldoi $tmp,$tmp,$tmp,15
2514	le?vperm $in1,$in1,$in1,$leperm
2515	vand $tmp,$tmp,$eighty7
2516	vxor $out1,$in1,$twk1
2517	vxor $tweak,$tweak,$tmp
2518
2519	lvx_u $in2,$x20,$inp
2520	andi. $taillen,$len,15
2521	vxor $twk2,$tweak,$rndkey0
2522	vsrab $tmp,$tweak,$seven # next tweak value
2523	vaddubm $tweak,$tweak,$tweak
2524	vsldoi $tmp,$tmp,$tmp,15
2525	le?vperm $in2,$in2,$in2,$leperm
2526	vand $tmp,$tmp,$eighty7
2527	vxor $out2,$in2,$twk2
2528	vxor $tweak,$tweak,$tmp
2529
2530	lvx_u $in3,$x30,$inp
2531	sub $len,$len,$taillen
2532	vxor $twk3,$tweak,$rndkey0
2533	vsrab $tmp,$tweak,$seven # next tweak value
2534	vaddubm $tweak,$tweak,$tweak
2535	vsldoi $tmp,$tmp,$tmp,15
2536	le?vperm $in3,$in3,$in3,$leperm
2537	vand $tmp,$tmp,$eighty7
2538	vxor $out3,$in3,$twk3
2539	vxor $tweak,$tweak,$tmp
2540
2541	lvx_u $in4,$x40,$inp
2542	subi $len,$len,0x60
2543	vxor $twk4,$tweak,$rndkey0
2544	vsrab $tmp,$tweak,$seven # next tweak value
2545	vaddubm $tweak,$tweak,$tweak
2546	vsldoi $tmp,$tmp,$tmp,15
2547	le?vperm $in4,$in4,$in4,$leperm
2548	vand $tmp,$tmp,$eighty7
2549	vxor $out4,$in4,$twk4
2550	vxor $tweak,$tweak,$tmp
2551
2552	lvx_u $in5,$x50,$inp
2553	addi $inp,$inp,0x60
2554	vxor $twk5,$tweak,$rndkey0
2555	vsrab $tmp,$tweak,$seven # next tweak value
2556	vaddubm $tweak,$tweak,$tweak
2557	vsldoi $tmp,$tmp,$tmp,15
2558	le?vperm $in5,$in5,$in5,$leperm
2559	vand $tmp,$tmp,$eighty7
2560	vxor $out5,$in5,$twk5
2561	vxor $tweak,$tweak,$tmp
2562
2563	vxor v31,v31,$rndkey0
2564	mtctr $rounds
2565	b Loop_xts_enc6x
2566
2567	.align 5
2568	Loop_xts_enc6x:
2569	vcipher $out0,$out0,v24
2570	vcipher $out1,$out1,v24
2571	vcipher $out2,$out2,v24
2572	vcipher $out3,$out3,v24
2573	vcipher $out4,$out4,v24
2574	vcipher $out5,$out5,v24
2575	lvx v24,$x20,$key_ # round[3]
2576	addi $key_,$key_,0x20
2577
2578	vcipher $out0,$out0,v25
2579	vcipher $out1,$out1,v25
2580	vcipher $out2,$out2,v25
2581	vcipher $out3,$out3,v25
2582	vcipher $out4,$out4,v25
2583	vcipher $out5,$out5,v25
2584	lvx v25,$x10,$key_ # round[4]
2585	bdnz Loop_xts_enc6x
2586
2587	subic $len,$len,96 # $len-=96
2588	vxor $in0,$twk0,v31 # xor with last round key
2589	vcipher $out0,$out0,v24
2590	vcipher $out1,$out1,v24
2591	vsrab $tmp,$tweak,$seven # next tweak value
2592	vxor $twk0,$tweak,$rndkey0
2593	vaddubm $tweak,$tweak,$tweak
2594	vcipher $out2,$out2,v24
2595	vcipher $out3,$out3,v24
2596	vsldoi $tmp,$tmp,$tmp,15
2597	vcipher $out4,$out4,v24
2598	vcipher $out5,$out5,v24
2599
2600	subfe. r0,r0,r0 # borrow?-1:0
2601	vand $tmp,$tmp,$eighty7
2602	vcipher $out0,$out0,v25
2603	vcipher $out1,$out1,v25
2604	vxor $tweak,$tweak,$tmp
2605	vcipher $out2,$out2,v25
2606	vcipher $out3,$out3,v25
2607	vxor $in1,$twk1,v31
2608	vsrab $tmp,$tweak,$seven # next tweak value
2609	vxor $twk1,$tweak,$rndkey0
2610	vcipher $out4,$out4,v25
2611	vcipher $out5,$out5,v25
2612
2613	and r0,r0,$len
2614	vaddubm $tweak,$tweak,$tweak
2615	vsldoi $tmp,$tmp,$tmp,15
2616	vcipher $out0,$out0,v26
2617	vcipher $out1,$out1,v26
2618	vand $tmp,$tmp,$eighty7
2619	vcipher $out2,$out2,v26
2620	vcipher $out3,$out3,v26
2621	vxor $tweak,$tweak,$tmp
2622	vcipher $out4,$out4,v26
2623	vcipher $out5,$out5,v26
2624
2625	add $inp,$inp,r0 # $inp is adjusted in such
2626	# way that at exit from the
2627	# loop inX-in5 are loaded
2628	# with last "words"
2629	vxor $in2,$twk2,v31
2630	vsrab $tmp,$tweak,$seven # next tweak value
2631	vxor $twk2,$tweak,$rndkey0
2632	vaddubm $tweak,$tweak,$tweak
2633	vcipher $out0,$out0,v27
2634	vcipher $out1,$out1,v27
2635	vsldoi $tmp,$tmp,$tmp,15
2636	vcipher $out2,$out2,v27
2637	vcipher $out3,$out3,v27
2638	vand $tmp,$tmp,$eighty7
2639	vcipher $out4,$out4,v27
2640	vcipher $out5,$out5,v27
2641
2642	addi $key_,$sp,$FRAME+15 # rewind $key_
2643	vxor $tweak,$tweak,$tmp
2644	vcipher $out0,$out0,v28
2645	vcipher $out1,$out1,v28
2646	vxor $in3,$twk3,v31
2647	vsrab $tmp,$tweak,$seven # next tweak value
2648	vxor $twk3,$tweak,$rndkey0
2649	vcipher $out2,$out2,v28
2650	vcipher $out3,$out3,v28
2651	vaddubm $tweak,$tweak,$tweak
2652	vsldoi $tmp,$tmp,$tmp,15
2653	vcipher $out4,$out4,v28
2654	vcipher $out5,$out5,v28
2655	lvx v24,$x00,$key_ # re-pre-load round[1]
2656	vand $tmp,$tmp,$eighty7
2657
2658	vcipher $out0,$out0,v29
2659	vcipher $out1,$out1,v29
2660	vxor $tweak,$tweak,$tmp
2661	vcipher $out2,$out2,v29
2662	vcipher $out3,$out3,v29
2663	vxor $in4,$twk4,v31
2664	vsrab $tmp,$tweak,$seven # next tweak value
2665	vxor $twk4,$tweak,$rndkey0
2666	vcipher $out4,$out4,v29
2667	vcipher $out5,$out5,v29
2668	lvx v25,$x10,$key_ # re-pre-load round[2]
2669	vaddubm $tweak,$tweak,$tweak
2670	vsldoi $tmp,$tmp,$tmp,15
2671
2672	vcipher $out0,$out0,v30
2673	vcipher $out1,$out1,v30
2674	vand $tmp,$tmp,$eighty7
2675	vcipher $out2,$out2,v30
2676	vcipher $out3,$out3,v30
2677	vxor $tweak,$tweak,$tmp
2678	vcipher $out4,$out4,v30
2679	vcipher $out5,$out5,v30
2680	vxor $in5,$twk5,v31
2681	vsrab $tmp,$tweak,$seven # next tweak value
2682	vxor $twk5,$tweak,$rndkey0
2683
2684	vcipherlast $out0,$out0,$in0
2685	lvx_u $in0,$x00,$inp # load next input block
2686	vaddubm $tweak,$tweak,$tweak
2687	vsldoi $tmp,$tmp,$tmp,15
2688	vcipherlast $out1,$out1,$in1
2689	lvx_u $in1,$x10,$inp
2690	vcipherlast $out2,$out2,$in2
2691	le?vperm $in0,$in0,$in0,$leperm
2692	lvx_u $in2,$x20,$inp
2693	vand $tmp,$tmp,$eighty7
2694	vcipherlast $out3,$out3,$in3
2695	le?vperm $in1,$in1,$in1,$leperm
2696	lvx_u $in3,$x30,$inp
2697	vcipherlast $out4,$out4,$in4
2698	le?vperm $in2,$in2,$in2,$leperm
2699	lvx_u $in4,$x40,$inp
2700	vxor $tweak,$tweak,$tmp
2701	vcipherlast $tmp,$out5,$in5 # last block might be needed
2702	# in stealing mode
2703	le?vperm $in3,$in3,$in3,$leperm
2704	lvx_u $in5,$x50,$inp
2705	addi $inp,$inp,0x60
2706	le?vperm $in4,$in4,$in4,$leperm
2707	le?vperm $in5,$in5,$in5,$leperm
2708
2709	le?vperm $out0,$out0,$out0,$leperm
2710	le?vperm $out1,$out1,$out1,$leperm
2711	stvx_u $out0,$x00,$out # store output
2712	vxor $out0,$in0,$twk0
2713	le?vperm $out2,$out2,$out2,$leperm
2714	stvx_u $out1,$x10,$out
2715	vxor $out1,$in1,$twk1
2716	le?vperm $out3,$out3,$out3,$leperm
2717	stvx_u $out2,$x20,$out
2718	vxor $out2,$in2,$twk2
2719	le?vperm $out4,$out4,$out4,$leperm
2720	stvx_u $out3,$x30,$out
2721	vxor $out3,$in3,$twk3
2722	le?vperm $out5,$tmp,$tmp,$leperm
2723	stvx_u $out4,$x40,$out
2724	vxor $out4,$in4,$twk4
2725	le?stvx_u $out5,$x50,$out
2726	be?stvx_u $tmp, $x50,$out
2727	vxor $out5,$in5,$twk5
2728	addi $out,$out,0x60
2729
2730	mtctr $rounds
2731	beq Loop_xts_enc6x # did $len-=96 borrow?
2732
2733	addic. $len,$len,0x60
2734	beq Lxts_enc6x_zero
2735	cmpwi $len,0x20
2736	blt Lxts_enc6x_one
2737	nop
2738	beq Lxts_enc6x_two
2739	cmpwi $len,0x40
2740	blt Lxts_enc6x_three
2741	nop
2742	beq Lxts_enc6x_four
2743
2744	Lxts_enc6x_five:
2745	vxor $out0,$in1,$twk0
2746	vxor $out1,$in2,$twk1
2747	vxor $out2,$in3,$twk2
2748	vxor $out3,$in4,$twk3
2749	vxor $out4,$in5,$twk4
2750
2751	bl _aesp8_xts_enc5x
2752
2753	le?vperm $out0,$out0,$out0,$leperm
2754	vmr $twk0,$twk5 # unused tweak
2755	le?vperm $out1,$out1,$out1,$leperm
2756	stvx_u $out0,$x00,$out # store output
2757	le?vperm $out2,$out2,$out2,$leperm
2758	stvx_u $out1,$x10,$out
2759	le?vperm $out3,$out3,$out3,$leperm
2760	stvx_u $out2,$x20,$out
2761	vxor $tmp,$out4,$twk5 # last block prep for stealing
2762	le?vperm $out4,$out4,$out4,$leperm
2763	stvx_u $out3,$x30,$out
2764	stvx_u $out4,$x40,$out
2765	addi $out,$out,0x50
2766	bne Lxts_enc6x_steal
2767	b Lxts_enc6x_done
2768
2769	.align 4
2770	Lxts_enc6x_four:
2771	vxor $out0,$in2,$twk0
2772	vxor $out1,$in3,$twk1
2773	vxor $out2,$in4,$twk2
2774	vxor $out3,$in5,$twk3
2775	vxor $out4,$out4,$out4
2776
2777	bl _aesp8_xts_enc5x
2778
2779	le?vperm $out0,$out0,$out0,$leperm
2780	vmr $twk0,$twk4 # unused tweak
2781	le?vperm $out1,$out1,$out1,$leperm
2782	stvx_u $out0,$x00,$out # store output
2783	le?vperm $out2,$out2,$out2,$leperm
2784	stvx_u $out1,$x10,$out
2785	vxor $tmp,$out3,$twk4 # last block prep for stealing
2786	le?vperm $out3,$out3,$out3,$leperm
2787	stvx_u $out2,$x20,$out
2788	stvx_u $out3,$x30,$out
2789	addi $out,$out,0x40
2790	bne Lxts_enc6x_steal
2791	b Lxts_enc6x_done
2792
2793	.align 4
2794	Lxts_enc6x_three:
2795	vxor $out0,$in3,$twk0
2796	vxor $out1,$in4,$twk1
2797	vxor $out2,$in5,$twk2
2798	vxor $out3,$out3,$out3
2799	vxor $out4,$out4,$out4
2800
2801	bl _aesp8_xts_enc5x
2802
2803	le?vperm $out0,$out0,$out0,$leperm
2804	vmr $twk0,$twk3 # unused tweak
2805	le?vperm $out1,$out1,$out1,$leperm
2806	stvx_u $out0,$x00,$out # store output
2807	vxor $tmp,$out2,$twk3 # last block prep for stealing
2808	le?vperm $out2,$out2,$out2,$leperm
2809	stvx_u $out1,$x10,$out
2810	stvx_u $out2,$x20,$out
2811	addi $out,$out,0x30
2812	bne Lxts_enc6x_steal
2813	b Lxts_enc6x_done
2814
2815	.align 4
2816	Lxts_enc6x_two:
2817	vxor $out0,$in4,$twk0
2818	vxor $out1,$in5,$twk1
2819	vxor $out2,$out2,$out2
2820	vxor $out3,$out3,$out3
2821	vxor $out4,$out4,$out4
2822
2823	bl _aesp8_xts_enc5x
2824
2825	le?vperm $out0,$out0,$out0,$leperm
2826	vmr $twk0,$twk2 # unused tweak
2827	vxor $tmp,$out1,$twk2 # last block prep for stealing
2828	le?vperm $out1,$out1,$out1,$leperm
2829	stvx_u $out0,$x00,$out # store output
2830	stvx_u $out1,$x10,$out
2831	addi $out,$out,0x20
2832	bne Lxts_enc6x_steal
2833	b Lxts_enc6x_done
2834
2835	.align 4
2836	Lxts_enc6x_one:
2837	vxor $out0,$in5,$twk0
2838	nop
2839	Loop_xts_enc1x:
2840	vcipher $out0,$out0,v24
2841	lvx v24,$x20,$key_ # round[3]
2842	addi $key_,$key_,0x20
2843
2844	vcipher $out0,$out0,v25
2845	lvx v25,$x10,$key_ # round[4]
2846	bdnz Loop_xts_enc1x
2847
2848	add $inp,$inp,$taillen
2849	cmpwi $taillen,0
2850	vcipher $out0,$out0,v24
2851
2852	subi $inp,$inp,16
2853	vcipher $out0,$out0,v25
2854
2855	lvsr $inpperm,0,$taillen
2856	vcipher $out0,$out0,v26
2857
2858	lvx_u $in0,0,$inp
2859	vcipher $out0,$out0,v27
2860
2861	addi $key_,$sp,$FRAME+15 # rewind $key_
2862	vcipher $out0,$out0,v28
2863	lvx v24,$x00,$key_ # re-pre-load round[1]
2864
2865	vcipher $out0,$out0,v29
2866	lvx v25,$x10,$key_ # re-pre-load round[2]
2867	vxor $twk0,$twk0,v31
2868
2869	le?vperm $in0,$in0,$in0,$leperm
2870	vcipher $out0,$out0,v30
2871
2872	vperm $in0,$in0,$in0,$inpperm
2873	vcipherlast $out0,$out0,$twk0
2874
2875	vmr $twk0,$twk1 # unused tweak
2876	vxor $tmp,$out0,$twk1 # last block prep for stealing
2877	le?vperm $out0,$out0,$out0,$leperm
2878	stvx_u $out0,$x00,$out # store output
2879	addi $out,$out,0x10
2880	bne Lxts_enc6x_steal
2881	b Lxts_enc6x_done
2882
2883	.align 4
2884	Lxts_enc6x_zero:
2885	cmpwi $taillen,0
2886	beq Lxts_enc6x_done
2887
2888	add $inp,$inp,$taillen
2889	subi $inp,$inp,16
2890	lvx_u $in0,0,$inp
2891	lvsr $inpperm,0,$taillen # $in5 is no more
2892	le?vperm $in0,$in0,$in0,$leperm
2893	vperm $in0,$in0,$in0,$inpperm
2894	vxor $tmp,$tmp,$twk0
2895	Lxts_enc6x_steal:
2896	vxor $in0,$in0,$twk0
2897	vxor $out0,$out0,$out0
2898	vspltisb $out1,-1
2899	vperm $out0,$out0,$out1,$inpperm
2900	vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember?
2901
2902	subi r30,$out,17
2903	subi $out,$out,16
2904	mtctr $taillen
2905	Loop_xts_enc6x_steal:
2906	lbzu r0,1(r30)
2907	stb r0,16(r30)
2908	bdnz Loop_xts_enc6x_steal
2909
2910	li $taillen,0
2911	mtctr $rounds
2912	b Loop_xts_enc1x # one more time...
2913
2914	.align 4
2915	Lxts_enc6x_done:
2916	${UCMP}i $ivp,0
2917	beq Lxts_enc6x_ret
2918
2919	vxor $tweak,$twk0,$rndkey0
2920	le?vperm $tweak,$tweak,$tweak,$leperm
2921	stvx_u $tweak,0,$ivp
2922
2923	Lxts_enc6x_ret:
2924	mtlr r11
2925	li r10,`$FRAME+15`
2926	li r11,`$FRAME+31`
2927	stvx $seven,r10,$sp # wipe copies of round keys
2928	addi r10,r10,32
2929	stvx $seven,r11,$sp
2930	addi r11,r11,32
2931	stvx $seven,r10,$sp
2932	addi r10,r10,32
2933	stvx $seven,r11,$sp
2934	addi r11,r11,32
2935	stvx $seven,r10,$sp
2936	addi r10,r10,32
2937	stvx $seven,r11,$sp
2938	addi r11,r11,32
2939	stvx $seven,r10,$sp
2940	addi r10,r10,32
2941	stvx $seven,r11,$sp
2942	addi r11,r11,32
2943
2944	mtspr 256,$vrsave
2945	lvx v20,r10,$sp # ABI says so
2946	addi r10,r10,32
2947	lvx v21,r11,$sp
2948	addi r11,r11,32
2949	lvx v22,r10,$sp
2950	addi r10,r10,32
2951	lvx v23,r11,$sp
2952	addi r11,r11,32
2953	lvx v24,r10,$sp
2954	addi r10,r10,32
2955	lvx v25,r11,$sp
2956	addi r11,r11,32
2957	lvx v26,r10,$sp
2958	addi r10,r10,32
2959	lvx v27,r11,$sp
2960	addi r11,r11,32
2961	lvx v28,r10,$sp
2962	addi r10,r10,32
2963	lvx v29,r11,$sp
2964	addi r11,r11,32
2965	lvx v30,r10,$sp
2966	lvx v31,r11,$sp
2967	$POP r26,`$FRAME+2116+0$SIZE_T`($sp)
2968	$POP r27,`$FRAME+2116+1$SIZE_T`($sp)
2969	$POP r28,`$FRAME+2116+2$SIZE_T`($sp)
2970	$POP r29,`$FRAME+2116+3$SIZE_T`($sp)
2971	$POP r30,`$FRAME+2116+4$SIZE_T`($sp)
2972	$POP r31,`$FRAME+2116+5$SIZE_T`($sp)
2973	addi $sp,$sp,`$FRAME+2116+6$SIZE_T`
2974	blr
2975	.long 0
2976	.byte 0,12,0x04,1,0x80,6,6,0
2977	.long 0
2978
2979	.align 5
2980	_aesp8_xts_enc5x:
2981	vcipher $out0,$out0,v24
2982	vcipher $out1,$out1,v24
2983	vcipher $out2,$out2,v24
2984	vcipher $out3,$out3,v24
2985	vcipher $out4,$out4,v24
2986	lvx v24,$x20,$key_ # round[3]
2987	addi $key_,$key_,0x20
2988
2989	vcipher $out0,$out0,v25
2990	vcipher $out1,$out1,v25
2991	vcipher $out2,$out2,v25
2992	vcipher $out3,$out3,v25
2993	vcipher $out4,$out4,v25
2994	lvx v25,$x10,$key_ # round[4]
2995	bdnz _aesp8_xts_enc5x
2996
2997	add $inp,$inp,$taillen
2998	cmpwi $taillen,0
2999	vcipher $out0,$out0,v24
3000	vcipher $out1,$out1,v24
3001	vcipher $out2,$out2,v24
3002	vcipher $out3,$out3,v24
3003	vcipher $out4,$out4,v24
3004
3005	subi $inp,$inp,16
3006	vcipher $out0,$out0,v25
3007	vcipher $out1,$out1,v25
3008	vcipher $out2,$out2,v25
3009	vcipher $out3,$out3,v25
3010	vcipher $out4,$out4,v25
3011	vxor $twk0,$twk0,v31
3012
3013	vcipher $out0,$out0,v26
3014	lvsr $inpperm,0,$taillen # $in5 is no more
3015	vcipher $out1,$out1,v26
3016	vcipher $out2,$out2,v26
3017	vcipher $out3,$out3,v26
3018	vcipher $out4,$out4,v26
3019	vxor $in1,$twk1,v31
3020
3021	vcipher $out0,$out0,v27
3022	lvx_u $in0,0,$inp
3023	vcipher $out1,$out1,v27
3024	vcipher $out2,$out2,v27
3025	vcipher $out3,$out3,v27
3026	vcipher $out4,$out4,v27
3027	vxor $in2,$twk2,v31
3028
3029	addi $key_,$sp,$FRAME+15 # rewind $key_
3030	vcipher $out0,$out0,v28
3031	vcipher $out1,$out1,v28
3032	vcipher $out2,$out2,v28
3033	vcipher $out3,$out3,v28
3034	vcipher $out4,$out4,v28
3035	lvx v24,$x00,$key_ # re-pre-load round[1]
3036	vxor $in3,$twk3,v31
3037
3038	vcipher $out0,$out0,v29
3039	le?vperm $in0,$in0,$in0,$leperm
3040	vcipher $out1,$out1,v29
3041	vcipher $out2,$out2,v29
3042	vcipher $out3,$out3,v29
3043	vcipher $out4,$out4,v29
3044	lvx v25,$x10,$key_ # re-pre-load round[2]
3045	vxor $in4,$twk4,v31
3046
3047	vcipher $out0,$out0,v30
3048	vperm $in0,$in0,$in0,$inpperm
3049	vcipher $out1,$out1,v30
3050	vcipher $out2,$out2,v30
3051	vcipher $out3,$out3,v30
3052	vcipher $out4,$out4,v30
3053
3054	vcipherlast $out0,$out0,$twk0
3055	vcipherlast $out1,$out1,$in1
3056	vcipherlast $out2,$out2,$in2
3057	vcipherlast $out3,$out3,$in3
3058	vcipherlast $out4,$out4,$in4
3059	blr
3060	.long 0
3061	.byte 0,12,0x14,0,0,0,0,0
3062
3063	.align 5
3064	_aesp8_xts_decrypt6x:
3065	$STU $sp,-`($FRAME+2116+6$SIZE_T)`($sp)
3066	mflr r11
3067	li r7,`$FRAME+8*16+15`
3068	li r3,`$FRAME+8*16+31`
3069	$PUSH r11,`$FRAME+2116+6$SIZE_T+$LRSAVE`($sp)
3070	stvx v20,r7,$sp # ABI says so
3071	addi r7,r7,32
3072	stvx v21,r3,$sp
3073	addi r3,r3,32
3074	stvx v22,r7,$sp
3075	addi r7,r7,32
3076	stvx v23,r3,$sp
3077	addi r3,r3,32
3078	stvx v24,r7,$sp
3079	addi r7,r7,32
3080	stvx v25,r3,$sp
3081	addi r3,r3,32
3082	stvx v26,r7,$sp
3083	addi r7,r7,32
3084	stvx v27,r3,$sp
3085	addi r3,r3,32
3086	stvx v28,r7,$sp
3087	addi r7,r7,32
3088	stvx v29,r3,$sp
3089	addi r3,r3,32
3090	stvx v30,r7,$sp
3091	stvx v31,r3,$sp
3092	li r0,-1
3093	stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
3094	li $x10,0x10
3095	$PUSH r26,`$FRAME+2116+0$SIZE_T`($sp)
3096	li $x20,0x20
3097	$PUSH r27,`$FRAME+2116+1$SIZE_T`($sp)
3098	li $x30,0x30
3099	$PUSH r28,`$FRAME+2116+2$SIZE_T`($sp)
3100	li $x40,0x40
3101	$PUSH r29,`$FRAME+2116+3$SIZE_T`($sp)
3102	li $x50,0x50
3103	$PUSH r30,`$FRAME+2116+4$SIZE_T`($sp)
3104	li $x60,0x60
3105	$PUSH r31,`$FRAME+2116+5$SIZE_T`($sp)
3106	li $x70,0x70
3107	mtspr 256,r0
3108
3109	subi $rounds,$rounds,3 # -4 in total
3110
3111	lvx $rndkey0,$x00,$key1 # load key schedule
3112	lvx v30,$x10,$key1
3113	addi $key1,$key1,0x20
3114	lvx v31,$x00,$key1
3115	?vperm $rndkey0,$rndkey0,v30,$keyperm
3116	addi $key_,$sp,$FRAME+15
3117	mtctr $rounds
3118
3119	Load_xts_dec_key:
3120	?vperm v24,v30,v31,$keyperm
3121	lvx v30,$x10,$key1
3122	addi $key1,$key1,0x20
3123	stvx v24,$x00,$key_ # off-load round[1]
3124	?vperm v25,v31,v30,$keyperm
3125	lvx v31,$x00,$key1
3126	stvx v25,$x10,$key_ # off-load round[2]
3127	addi $key_,$key_,0x20
3128	bdnz Load_xts_dec_key
3129
3130	lvx v26,$x10,$key1
3131	?vperm v24,v30,v31,$keyperm
3132	lvx v27,$x20,$key1
3133	stvx v24,$x00,$key_ # off-load round[3]
3134	?vperm v25,v31,v26,$keyperm
3135	lvx v28,$x30,$key1
3136	stvx v25,$x10,$key_ # off-load round[4]
3137	addi $key_,$sp,$FRAME+15 # rewind $key_
3138	?vperm v26,v26,v27,$keyperm
3139	lvx v29,$x40,$key1
3140	?vperm v27,v27,v28,$keyperm
3141	lvx v30,$x50,$key1
3142	?vperm v28,v28,v29,$keyperm
3143	lvx v31,$x60,$key1
3144	?vperm v29,v29,v30,$keyperm
3145	lvx $twk5,$x70,$key1 # borrow $twk5
3146	?vperm v30,v30,v31,$keyperm
3147	lvx v24,$x00,$key_ # pre-load round[1]
3148	?vperm v31,v31,$twk5,$keyperm
3149	lvx v25,$x10,$key_ # pre-load round[2]
3150
3151	vperm $in0,$inout,$inptail,$inpperm
3152	subi $inp,$inp,31 # undo "caller"
3153	vxor $twk0,$tweak,$rndkey0
3154	vsrab $tmp,$tweak,$seven # next tweak value
3155	vaddubm $tweak,$tweak,$tweak
3156	vsldoi $tmp,$tmp,$tmp,15
3157	vand $tmp,$tmp,$eighty7
3158	vxor $out0,$in0,$twk0
3159	vxor $tweak,$tweak,$tmp
3160
3161	lvx_u $in1,$x10,$inp
3162	vxor $twk1,$tweak,$rndkey0
3163	vsrab $tmp,$tweak,$seven # next tweak value
3164	vaddubm $tweak,$tweak,$tweak
3165	vsldoi $tmp,$tmp,$tmp,15
3166	le?vperm $in1,$in1,$in1,$leperm
3167	vand $tmp,$tmp,$eighty7
3168	vxor $out1,$in1,$twk1
3169	vxor $tweak,$tweak,$tmp
3170
3171	lvx_u $in2,$x20,$inp
3172	andi. $taillen,$len,15
3173	vxor $twk2,$tweak,$rndkey0
3174	vsrab $tmp,$tweak,$seven # next tweak value
3175	vaddubm $tweak,$tweak,$tweak
3176	vsldoi $tmp,$tmp,$tmp,15
3177	le?vperm $in2,$in2,$in2,$leperm
3178	vand $tmp,$tmp,$eighty7
3179	vxor $out2,$in2,$twk2
3180	vxor $tweak,$tweak,$tmp
3181
3182	lvx_u $in3,$x30,$inp
3183	sub $len,$len,$taillen
3184	vxor $twk3,$tweak,$rndkey0
3185	vsrab $tmp,$tweak,$seven # next tweak value
3186	vaddubm $tweak,$tweak,$tweak
3187	vsldoi $tmp,$tmp,$tmp,15
3188	le?vperm $in3,$in3,$in3,$leperm
3189	vand $tmp,$tmp,$eighty7
3190	vxor $out3,$in3,$twk3
3191	vxor $tweak,$tweak,$tmp
3192
3193	lvx_u $in4,$x40,$inp
3194	subi $len,$len,0x60
3195	vxor $twk4,$tweak,$rndkey0
3196	vsrab $tmp,$tweak,$seven # next tweak value
3197	vaddubm $tweak,$tweak,$tweak
3198	vsldoi $tmp,$tmp,$tmp,15
3199	le?vperm $in4,$in4,$in4,$leperm
3200	vand $tmp,$tmp,$eighty7
3201	vxor $out4,$in4,$twk4
3202	vxor $tweak,$tweak,$tmp
3203
3204	lvx_u $in5,$x50,$inp
3205	addi $inp,$inp,0x60
3206	vxor $twk5,$tweak,$rndkey0
3207	vsrab $tmp,$tweak,$seven # next tweak value
3208	vaddubm $tweak,$tweak,$tweak
3209	vsldoi $tmp,$tmp,$tmp,15
3210	le?vperm $in5,$in5,$in5,$leperm
3211	vand $tmp,$tmp,$eighty7
3212	vxor $out5,$in5,$twk5
3213	vxor $tweak,$tweak,$tmp
3214
3215	vxor v31,v31,$rndkey0
3216	mtctr $rounds
3217	b Loop_xts_dec6x
3218
3219	.align 5
3220	Loop_xts_dec6x:
3221	vncipher $out0,$out0,v24
3222	vncipher $out1,$out1,v24
3223	vncipher $out2,$out2,v24
3224	vncipher $out3,$out3,v24
3225	vncipher $out4,$out4,v24
3226	vncipher $out5,$out5,v24
3227	lvx v24,$x20,$key_ # round[3]
3228	addi $key_,$key_,0x20
3229
3230	vncipher $out0,$out0,v25
3231	vncipher $out1,$out1,v25
3232	vncipher $out2,$out2,v25
3233	vncipher $out3,$out3,v25
3234	vncipher $out4,$out4,v25
3235	vncipher $out5,$out5,v25
3236	lvx v25,$x10,$key_ # round[4]
3237	bdnz Loop_xts_dec6x
3238
3239	subic $len,$len,96 # $len-=96
3240	vxor $in0,$twk0,v31 # xor with last round key
3241	vncipher $out0,$out0,v24
3242	vncipher $out1,$out1,v24
3243	vsrab $tmp,$tweak,$seven # next tweak value
3244	vxor $twk0,$tweak,$rndkey0
3245	vaddubm $tweak,$tweak,$tweak
3246	vncipher $out2,$out2,v24
3247	vncipher $out3,$out3,v24
3248	vsldoi $tmp,$tmp,$tmp,15
3249	vncipher $out4,$out4,v24
3250	vncipher $out5,$out5,v24
3251
3252	subfe. r0,r0,r0 # borrow?-1:0
3253	vand $tmp,$tmp,$eighty7
3254	vncipher $out0,$out0,v25
3255	vncipher $out1,$out1,v25
3256	vxor $tweak,$tweak,$tmp
3257	vncipher $out2,$out2,v25
3258	vncipher $out3,$out3,v25
3259	vxor $in1,$twk1,v31
3260	vsrab $tmp,$tweak,$seven # next tweak value
3261	vxor $twk1,$tweak,$rndkey0
3262	vncipher $out4,$out4,v25
3263	vncipher $out5,$out5,v25
3264
3265	and r0,r0,$len
3266	vaddubm $tweak,$tweak,$tweak
3267	vsldoi $tmp,$tmp,$tmp,15
3268	vncipher $out0,$out0,v26
3269	vncipher $out1,$out1,v26
3270	vand $tmp,$tmp,$eighty7
3271	vncipher $out2,$out2,v26
3272	vncipher $out3,$out3,v26
3273	vxor $tweak,$tweak,$tmp
3274	vncipher $out4,$out4,v26
3275	vncipher $out5,$out5,v26
3276
3277	add $inp,$inp,r0 # $inp is adjusted in such
3278	# way that at exit from the
3279	# loop inX-in5 are loaded
3280	# with last "words"
3281	vxor $in2,$twk2,v31
3282	vsrab $tmp,$tweak,$seven # next tweak value
3283	vxor $twk2,$tweak,$rndkey0
3284	vaddubm $tweak,$tweak,$tweak
3285	vncipher $out0,$out0,v27
3286	vncipher $out1,$out1,v27
3287	vsldoi $tmp,$tmp,$tmp,15
3288	vncipher $out2,$out2,v27
3289	vncipher $out3,$out3,v27
3290	vand $tmp,$tmp,$eighty7
3291	vncipher $out4,$out4,v27
3292	vncipher $out5,$out5,v27
3293
3294	addi $key_,$sp,$FRAME+15 # rewind $key_
3295	vxor $tweak,$tweak,$tmp
3296	vncipher $out0,$out0,v28
3297	vncipher $out1,$out1,v28
3298	vxor $in3,$twk3,v31
3299	vsrab $tmp,$tweak,$seven # next tweak value
3300	vxor $twk3,$tweak,$rndkey0
3301	vncipher $out2,$out2,v28
3302	vncipher $out3,$out3,v28
3303	vaddubm $tweak,$tweak,$tweak
3304	vsldoi $tmp,$tmp,$tmp,15
3305	vncipher $out4,$out4,v28
3306	vncipher $out5,$out5,v28
3307	lvx v24,$x00,$key_ # re-pre-load round[1]
3308	vand $tmp,$tmp,$eighty7
3309
3310	vncipher $out0,$out0,v29
3311	vncipher $out1,$out1,v29
3312	vxor $tweak,$tweak,$tmp
3313	vncipher $out2,$out2,v29
3314	vncipher $out3,$out3,v29
3315	vxor $in4,$twk4,v31
3316	vsrab $tmp,$tweak,$seven # next tweak value
3317	vxor $twk4,$tweak,$rndkey0
3318	vncipher $out4,$out4,v29
3319	vncipher $out5,$out5,v29
3320	lvx v25,$x10,$key_ # re-pre-load round[2]
3321	vaddubm $tweak,$tweak,$tweak
3322	vsldoi $tmp,$tmp,$tmp,15
3323
3324	vncipher $out0,$out0,v30
3325	vncipher $out1,$out1,v30
3326	vand $tmp,$tmp,$eighty7
3327	vncipher $out2,$out2,v30
3328	vncipher $out3,$out3,v30
3329	vxor $tweak,$tweak,$tmp
3330	vncipher $out4,$out4,v30
3331	vncipher $out5,$out5,v30
3332	vxor $in5,$twk5,v31
3333	vsrab $tmp,$tweak,$seven # next tweak value
3334	vxor $twk5,$tweak,$rndkey0
3335
3336	vncipherlast $out0,$out0,$in0
3337	lvx_u $in0,$x00,$inp # load next input block
3338	vaddubm $tweak,$tweak,$tweak
3339	vsldoi $tmp,$tmp,$tmp,15
3340	vncipherlast $out1,$out1,$in1
3341	lvx_u $in1,$x10,$inp
3342	vncipherlast $out2,$out2,$in2
3343	le?vperm $in0,$in0,$in0,$leperm
3344	lvx_u $in2,$x20,$inp
3345	vand $tmp,$tmp,$eighty7
3346	vncipherlast $out3,$out3,$in3
3347	le?vperm $in1,$in1,$in1,$leperm
3348	lvx_u $in3,$x30,$inp
3349	vncipherlast $out4,$out4,$in4
3350	le?vperm $in2,$in2,$in2,$leperm
3351	lvx_u $in4,$x40,$inp
3352	vxor $tweak,$tweak,$tmp
3353	vncipherlast $out5,$out5,$in5
3354	le?vperm $in3,$in3,$in3,$leperm
3355	lvx_u $in5,$x50,$inp
3356	addi $inp,$inp,0x60
3357	le?vperm $in4,$in4,$in4,$leperm
3358	le?vperm $in5,$in5,$in5,$leperm
3359
3360	le?vperm $out0,$out0,$out0,$leperm
3361	le?vperm $out1,$out1,$out1,$leperm
3362	stvx_u $out0,$x00,$out # store output
3363	vxor $out0,$in0,$twk0
3364	le?vperm $out2,$out2,$out2,$leperm
3365	stvx_u $out1,$x10,$out
3366	vxor $out1,$in1,$twk1
3367	le?vperm $out3,$out3,$out3,$leperm
3368	stvx_u $out2,$x20,$out
3369	vxor $out2,$in2,$twk2
3370	le?vperm $out4,$out4,$out4,$leperm
3371	stvx_u $out3,$x30,$out
3372	vxor $out3,$in3,$twk3
3373	le?vperm $out5,$out5,$out5,$leperm
3374	stvx_u $out4,$x40,$out
3375	vxor $out4,$in4,$twk4
3376	stvx_u $out5,$x50,$out
3377	vxor $out5,$in5,$twk5
3378	addi $out,$out,0x60
3379
3380	mtctr $rounds
3381	beq Loop_xts_dec6x # did $len-=96 borrow?
3382
3383	addic. $len,$len,0x60
3384	beq Lxts_dec6x_zero
3385	cmpwi $len,0x20
3386	blt Lxts_dec6x_one
3387	nop
3388	beq Lxts_dec6x_two
3389	cmpwi $len,0x40
3390	blt Lxts_dec6x_three
3391	nop
3392	beq Lxts_dec6x_four
3393
3394	Lxts_dec6x_five:
3395	vxor $out0,$in1,$twk0
3396	vxor $out1,$in2,$twk1
3397	vxor $out2,$in3,$twk2
3398	vxor $out3,$in4,$twk3
3399	vxor $out4,$in5,$twk4
3400
3401	bl _aesp8_xts_dec5x
3402
3403	le?vperm $out0,$out0,$out0,$leperm
3404	vmr $twk0,$twk5 # unused tweak
3405	vxor $twk1,$tweak,$rndkey0
3406	le?vperm $out1,$out1,$out1,$leperm
3407	stvx_u $out0,$x00,$out # store output
3408	vxor $out0,$in0,$twk1
3409	le?vperm $out2,$out2,$out2,$leperm
3410	stvx_u $out1,$x10,$out
3411	le?vperm $out3,$out3,$out3,$leperm
3412	stvx_u $out2,$x20,$out
3413	le?vperm $out4,$out4,$out4,$leperm
3414	stvx_u $out3,$x30,$out
3415	stvx_u $out4,$x40,$out
3416	addi $out,$out,0x50
3417	bne Lxts_dec6x_steal
3418	b Lxts_dec6x_done
3419
3420	.align 4
3421	Lxts_dec6x_four:
3422	vxor $out0,$in2,$twk0
3423	vxor $out1,$in3,$twk1
3424	vxor $out2,$in4,$twk2
3425	vxor $out3,$in5,$twk3
3426	vxor $out4,$out4,$out4
3427
3428	bl _aesp8_xts_dec5x
3429
3430	le?vperm $out0,$out0,$out0,$leperm
3431	vmr $twk0,$twk4 # unused tweak
3432	vmr $twk1,$twk5
3433	le?vperm $out1,$out1,$out1,$leperm
3434	stvx_u $out0,$x00,$out # store output
3435	vxor $out0,$in0,$twk5
3436	le?vperm $out2,$out2,$out2,$leperm
3437	stvx_u $out1,$x10,$out
3438	le?vperm $out3,$out3,$out3,$leperm
3439	stvx_u $out2,$x20,$out
3440	stvx_u $out3,$x30,$out
3441	addi $out,$out,0x40
3442	bne Lxts_dec6x_steal
3443	b Lxts_dec6x_done
3444
3445	.align 4
3446	Lxts_dec6x_three:
3447	vxor $out0,$in3,$twk0
3448	vxor $out1,$in4,$twk1
3449	vxor $out2,$in5,$twk2
3450	vxor $out3,$out3,$out3
3451	vxor $out4,$out4,$out4
3452
3453	bl _aesp8_xts_dec5x
3454
3455	le?vperm $out0,$out0,$out0,$leperm
3456	vmr $twk0,$twk3 # unused tweak
3457	vmr $twk1,$twk4
3458	le?vperm $out1,$out1,$out1,$leperm
3459	stvx_u $out0,$x00,$out # store output
3460	vxor $out0,$in0,$twk4
3461	le?vperm $out2,$out2,$out2,$leperm
3462	stvx_u $out1,$x10,$out
3463	stvx_u $out2,$x20,$out
3464	addi $out,$out,0x30
3465	bne Lxts_dec6x_steal
3466	b Lxts_dec6x_done
3467
3468	.align 4
3469	Lxts_dec6x_two:
3470	vxor $out0,$in4,$twk0
3471	vxor $out1,$in5,$twk1
3472	vxor $out2,$out2,$out2
3473	vxor $out3,$out3,$out3
3474	vxor $out4,$out4,$out4
3475
3476	bl _aesp8_xts_dec5x
3477
3478	le?vperm $out0,$out0,$out0,$leperm
3479	vmr $twk0,$twk2 # unused tweak
3480	vmr $twk1,$twk3
3481	le?vperm $out1,$out1,$out1,$leperm
3482	stvx_u $out0,$x00,$out # store output
3483	vxor $out0,$in0,$twk3
3484	stvx_u $out1,$x10,$out
3485	addi $out,$out,0x20
3486	bne Lxts_dec6x_steal
3487	b Lxts_dec6x_done
3488
3489	.align 4
3490	Lxts_dec6x_one:
3491	vxor $out0,$in5,$twk0
3492	nop
3493	Loop_xts_dec1x:
3494	vncipher $out0,$out0,v24
3495	lvx v24,$x20,$key_ # round[3]
3496	addi $key_,$key_,0x20
3497
3498	vncipher $out0,$out0,v25
3499	lvx v25,$x10,$key_ # round[4]
3500	bdnz Loop_xts_dec1x
3501
3502	subi r0,$taillen,1
3503	vncipher $out0,$out0,v24
3504
3505	andi. r0,r0,16
3506	cmpwi $taillen,0
3507	vncipher $out0,$out0,v25
3508
3509	sub $inp,$inp,r0
3510	vncipher $out0,$out0,v26
3511
3512	lvx_u $in0,0,$inp
3513	vncipher $out0,$out0,v27
3514
3515	addi $key_,$sp,$FRAME+15 # rewind $key_
3516	vncipher $out0,$out0,v28
3517	lvx v24,$x00,$key_ # re-pre-load round[1]
3518
3519	vncipher $out0,$out0,v29
3520	lvx v25,$x10,$key_ # re-pre-load round[2]
3521	vxor $twk0,$twk0,v31
3522
3523	le?vperm $in0,$in0,$in0,$leperm
3524	vncipher $out0,$out0,v30
3525
3526	mtctr $rounds
3527	vncipherlast $out0,$out0,$twk0
3528
3529	vmr $twk0,$twk1 # unused tweak
3530	vmr $twk1,$twk2
3531	le?vperm $out0,$out0,$out0,$leperm
3532	stvx_u $out0,$x00,$out # store output
3533	addi $out,$out,0x10
3534	vxor $out0,$in0,$twk2
3535	bne Lxts_dec6x_steal
3536	b Lxts_dec6x_done
3537
3538	.align 4
3539	Lxts_dec6x_zero:
3540	cmpwi $taillen,0
3541	beq Lxts_dec6x_done
3542
3543	lvx_u $in0,0,$inp
3544	le?vperm $in0,$in0,$in0,$leperm
3545	vxor $out0,$in0,$twk1
3546	Lxts_dec6x_steal:
3547	vncipher $out0,$out0,v24
3548	lvx v24,$x20,$key_ # round[3]
3549	addi $key_,$key_,0x20
3550
3551	vncipher $out0,$out0,v25
3552	lvx v25,$x10,$key_ # round[4]
3553	bdnz Lxts_dec6x_steal
3554
3555	add $inp,$inp,$taillen
3556	vncipher $out0,$out0,v24
3557
3558	cmpwi $taillen,0
3559	vncipher $out0,$out0,v25
3560
3561	lvx_u $in0,0,$inp
3562	vncipher $out0,$out0,v26
3563
3564	lvsr $inpperm,0,$taillen # $in5 is no more
3565	vncipher $out0,$out0,v27
3566
3567	addi $key_,$sp,$FRAME+15 # rewind $key_
3568	vncipher $out0,$out0,v28
3569	lvx v24,$x00,$key_ # re-pre-load round[1]
3570
3571	vncipher $out0,$out0,v29
3572	lvx v25,$x10,$key_ # re-pre-load round[2]
3573	vxor $twk1,$twk1,v31
3574
3575	le?vperm $in0,$in0,$in0,$leperm
3576	vncipher $out0,$out0,v30
3577
3578	vperm $in0,$in0,$in0,$inpperm
3579	vncipherlast $tmp,$out0,$twk1
3580
3581	le?vperm $out0,$tmp,$tmp,$leperm
3582	le?stvx_u $out0,0,$out
3583	be?stvx_u $tmp,0,$out
3584
3585	vxor $out0,$out0,$out0
3586	vspltisb $out1,-1
3587	vperm $out0,$out0,$out1,$inpperm
3588	vsel $out0,$in0,$tmp,$out0
3589	vxor $out0,$out0,$twk0
3590
3591	subi r30,$out,1
3592	mtctr $taillen
3593	Loop_xts_dec6x_steal:
3594	lbzu r0,1(r30)
3595	stb r0,16(r30)
3596	bdnz Loop_xts_dec6x_steal
3597
3598	li $taillen,0
3599	mtctr $rounds
3600	b Loop_xts_dec1x # one more time...
3601
3602	.align 4
3603	Lxts_dec6x_done:
3604	${UCMP}i $ivp,0
3605	beq Lxts_dec6x_ret
3606
3607	vxor $tweak,$twk0,$rndkey0
3608	le?vperm $tweak,$tweak,$tweak,$leperm
3609	stvx_u $tweak,0,$ivp
3610
3611	Lxts_dec6x_ret:
3612	mtlr r11
3613	li r10,`$FRAME+15`
3614	li r11,`$FRAME+31`
3615	stvx $seven,r10,$sp # wipe copies of round keys
3616	addi r10,r10,32
3617	stvx $seven,r11,$sp
3618	addi r11,r11,32
3619	stvx $seven,r10,$sp
3620	addi r10,r10,32
3621	stvx $seven,r11,$sp
3622	addi r11,r11,32
3623	stvx $seven,r10,$sp
3624	addi r10,r10,32
3625	stvx $seven,r11,$sp
3626	addi r11,r11,32
3627	stvx $seven,r10,$sp
3628	addi r10,r10,32
3629	stvx $seven,r11,$sp
3630	addi r11,r11,32
3631
3632	mtspr 256,$vrsave
3633	lvx v20,r10,$sp # ABI says so
3634	addi r10,r10,32
3635	lvx v21,r11,$sp
3636	addi r11,r11,32
3637	lvx v22,r10,$sp
3638	addi r10,r10,32
3639	lvx v23,r11,$sp
3640	addi r11,r11,32
3641	lvx v24,r10,$sp
3642	addi r10,r10,32
3643	lvx v25,r11,$sp
3644	addi r11,r11,32
3645	lvx v26,r10,$sp
3646	addi r10,r10,32
3647	lvx v27,r11,$sp
3648	addi r11,r11,32
3649	lvx v28,r10,$sp
3650	addi r10,r10,32
3651	lvx v29,r11,$sp
3652	addi r11,r11,32
3653	lvx v30,r10,$sp
3654	lvx v31,r11,$sp
3655	$POP r26,`$FRAME+2116+0$SIZE_T`($sp)
3656	$POP r27,`$FRAME+2116+1$SIZE_T`($sp)
3657	$POP r28,`$FRAME+2116+2$SIZE_T`($sp)
3658	$POP r29,`$FRAME+2116+3$SIZE_T`($sp)
3659	$POP r30,`$FRAME+2116+4$SIZE_T`($sp)
3660	$POP r31,`$FRAME+2116+5$SIZE_T`($sp)
3661	addi $sp,$sp,`$FRAME+2116+6$SIZE_T`
3662	blr
3663	.long 0
3664	.byte 0,12,0x04,1,0x80,6,6,0
3665	.long 0
3666
3667	.align 5
3668	_aesp8_xts_dec5x:
3669	vncipher $out0,$out0,v24
3670	vncipher $out1,$out1,v24
3671	vncipher $out2,$out2,v24
3672	vncipher $out3,$out3,v24
3673	vncipher $out4,$out4,v24
3674	lvx v24,$x20,$key_ # round[3]
3675	addi $key_,$key_,0x20
3676
3677	vncipher $out0,$out0,v25
3678	vncipher $out1,$out1,v25
3679	vncipher $out2,$out2,v25
3680	vncipher $out3,$out3,v25
3681	vncipher $out4,$out4,v25
3682	lvx v25,$x10,$key_ # round[4]
3683	bdnz _aesp8_xts_dec5x
3684
3685	subi r0,$taillen,1
3686	vncipher $out0,$out0,v24
3687	vncipher $out1,$out1,v24
3688	vncipher $out2,$out2,v24
3689	vncipher $out3,$out3,v24
3690	vncipher $out4,$out4,v24
3691
3692	andi. r0,r0,16
3693	cmpwi $taillen,0
3694	vncipher $out0,$out0,v25
3695	vncipher $out1,$out1,v25
3696	vncipher $out2,$out2,v25
3697	vncipher $out3,$out3,v25
3698	vncipher $out4,$out4,v25
3699	vxor $twk0,$twk0,v31
3700
3701	sub $inp,$inp,r0
3702	vncipher $out0,$out0,v26
3703	vncipher $out1,$out1,v26
3704	vncipher $out2,$out2,v26
3705	vncipher $out3,$out3,v26
3706	vncipher $out4,$out4,v26
3707	vxor $in1,$twk1,v31
3708
3709	vncipher $out0,$out0,v27
3710	lvx_u $in0,0,$inp
3711	vncipher $out1,$out1,v27
3712	vncipher $out2,$out2,v27
3713	vncipher $out3,$out3,v27
3714	vncipher $out4,$out4,v27
3715	vxor $in2,$twk2,v31
3716
3717	addi $key_,$sp,$FRAME+15 # rewind $key_
3718	vncipher $out0,$out0,v28
3719	vncipher $out1,$out1,v28
3720	vncipher $out2,$out2,v28
3721	vncipher $out3,$out3,v28
3722	vncipher $out4,$out4,v28
3723	lvx v24,$x00,$key_ # re-pre-load round[1]
3724	vxor $in3,$twk3,v31
3725
3726	vncipher $out0,$out0,v29
3727	le?vperm $in0,$in0,$in0,$leperm
3728	vncipher $out1,$out1,v29
3729	vncipher $out2,$out2,v29
3730	vncipher $out3,$out3,v29
3731	vncipher $out4,$out4,v29
3732	lvx v25,$x10,$key_ # re-pre-load round[2]
3733	vxor $in4,$twk4,v31
3734
3735	vncipher $out0,$out0,v30
3736	vncipher $out1,$out1,v30
3737	vncipher $out2,$out2,v30
3738	vncipher $out3,$out3,v30
3739	vncipher $out4,$out4,v30
3740
3741	vncipherlast $out0,$out0,$twk0
3742	vncipherlast $out1,$out1,$in1
3743	vncipherlast $out2,$out2,$in2
3744	vncipherlast $out3,$out3,$in3
3745	vncipherlast $out4,$out4,$in4
3746	mtctr $rounds
3747	blr
3748	.long 0
3749	.byte 0,12,0x14,0,0,0,0,0
3750	___
3751	}} }}}
3752
3753	my $consts=1;
3754	foreach(split("\n",$code)) {
3755	s/\`([^\`]*)\`/eval($1)/geo;
3756
3757	# constants table endian-specific conversion
3758	if ($consts && m/\.(long\|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3759	my $conv=$3;
3760	my @bytes=();
3761
3762	# convert to endian-agnostic format
3763	if ($1 eq "long") {
3764	foreach (split(/,\s*/,$2)) {
3765	my $l = /^0/?oct:int;
3766	push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
3767	}
3768	} else {
3769	@bytes = map(/^0/?oct:int,split(/,\s*/,$2));
3770	}
3771
3772	# little-endian conversion
3773	if ($flavour =~ /le$/o) {
3774	SWITCH: for($conv) {
3775	/\?inv/ && do { @bytes=map($_^0xf,@bytes); last; };
3776	/\?rev/ && do { @bytes=reverse(@bytes); last; };
3777	}
3778	}
3779
3780	#emit
3781	print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
3782	next;
3783	}
3784	$consts=0 if (m/Lconsts:/o); # end of table
3785
3786	# instructions prefixed with '?' are endian-specific and need
3787	# to be adjusted accordingly...
3788	if ($flavour =~ /le$/o) { # little-endian
3789	s/le\?//o or
3790	s/be\?/#be#/o or
3791	s/\?lvsr/lvsl/o or
3792	s/\?lvsl/lvsr/o or
3793	s/\?(vperm\s+v[0-9]+,\s)(v[0-9]+,\s)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3794	s/\?(vsldoi\s+v[0-9]+,\s)(v[0-9]+,)\s(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3795	s/\?(vspltw\s+v[0-9]+,\s)(v[0-9]+,)\s([0-9])/$1$2 3-$3/o;
3796	} else { # big-endian
3797	s/le\?/#le#/o or
3798	s/be\?//o or
3799	s/\?([a-z]+)/$1/o;
3800	}
3801
3802	print $_,"\n";
3803	}
3804
3805	close STDOUT;

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/libs/openssl-1.1.0g/crypto/aes/asm/aesp8-ppc.pl@ 69890

Download in other formats: