| 1 | #! /usr/bin/env perl
|
|---|
| 2 | # Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
|
|---|
| 3 | #
|
|---|
| 4 | # Licensed under the OpenSSL license (the "License"). You may not use
|
|---|
| 5 | # this file except in compliance with the License. You can obtain a copy
|
|---|
| 6 | # in the file LICENSE in the source distribution or at
|
|---|
| 7 | # https://www.openssl.org/source/license.html
|
|---|
| 8 |
|
|---|
| 9 |
|
|---|
| 10 | # ====================================================================
|
|---|
| 11 | # Written by Andy Polyakov <[email protected]> for the OpenSSL
|
|---|
| 12 | # project. The module is, however, dual licensed under OpenSSL and
|
|---|
| 13 | # CRYPTOGAMS licenses depending on where you obtain it. For further
|
|---|
| 14 | # details see http://www.openssl.org/~appro/cryptogams/.
|
|---|
| 15 | # ====================================================================
|
|---|
| 16 | #
|
|---|
| 17 | # March 2010
|
|---|
| 18 | #
|
|---|
| 19 | # The module implements "4-bit" GCM GHASH function and underlying
|
|---|
| 20 | # single multiplication operation in GF(2^128). "4-bit" means that it
|
|---|
| 21 | # uses 256 bytes per-key table [+128 bytes shared table]. Streamed
|
|---|
| 22 | # GHASH performance was measured to be 6.67 cycles per processed byte
|
|---|
| 23 | # on Itanium 2, which is >90% better than Microsoft compiler generated
|
|---|
| 24 | # code. To anchor to something else sha1-ia64.pl module processes one
|
|---|
| 25 | # byte in 5.7 cycles. On Itanium GHASH should run at ~8.5 cycles per
|
|---|
| 26 | # byte.
|
|---|
| 27 |
|
|---|
| 28 | # September 2010
|
|---|
| 29 | #
|
|---|
| 30 | # It was originally thought that it makes lesser sense to implement
|
|---|
| 31 | # "528B" variant on Itanium 2 for following reason. Because number of
|
|---|
| 32 | # functional units is naturally limited, it appeared impossible to
|
|---|
| 33 | # implement "528B" loop in 4 cycles, only in 5. This would mean that
|
|---|
| 34 | # theoretically performance improvement couldn't be more than 20%.
|
|---|
| 35 | # But occasionally you prove yourself wrong:-) I figured out a way to
|
|---|
| 36 | # fold couple of instructions and having freed yet another instruction
|
|---|
| 37 | # slot by unrolling the loop... Resulting performance is 4.45 cycles
|
|---|
| 38 | # per processed byte and 50% better than "256B" version. On original
|
|---|
| 39 | # Itanium performance should remain the same as the "256B" version,
|
|---|
| 40 | # i.e. ~8.5 cycles.
|
|---|
| 41 |
|
|---|
| 42 | $output=pop and (open STDOUT,">$output" or die "can't open $output: $!");
|
|---|
| 43 |
|
|---|
| 44 | if ($^O eq "hpux") {
|
|---|
| 45 | $ADDP="addp4";
|
|---|
| 46 | for (@ARGV) { $ADDP="add" if (/[\+DD|\-mlp]64/); }
|
|---|
| 47 | } else { $ADDP="add"; }
|
|---|
| 48 | for (@ARGV) { $big_endian=1 if (/\-DB_ENDIAN/);
|
|---|
| 49 | $big_endian=0 if (/\-DL_ENDIAN/); }
|
|---|
| 50 | if (!defined($big_endian))
|
|---|
| 51 | { $big_endian=(unpack('L',pack('N',1))==1); }
|
|---|
| 52 |
|
|---|
| 53 | sub loop() {
|
|---|
| 54 | my $label=shift;
|
|---|
| 55 | my ($p16,$p17)=(shift)?("p63","p63"):("p16","p17"); # mask references to inp
|
|---|
| 56 |
|
|---|
| 57 | # Loop is scheduled for 6 ticks on Itanium 2 and 8 on Itanium, i.e.
|
|---|
| 58 | # in scalable manner;-) Naturally assuming data in L1 cache...
|
|---|
| 59 | # Special note about 'dep' instruction, which is used to construct
|
|---|
| 60 | # &rem_4bit[Zlo&0xf]. It works, because rem_4bit is aligned at 128
|
|---|
| 61 | # bytes boundary and lower 7 bits of its address are guaranteed to
|
|---|
| 62 | # be zero.
|
|---|
| 63 | $code.=<<___;
|
|---|
| 64 | $label:
|
|---|
| 65 | { .mfi; (p18) ld8 Hlo=[Hi[1]],-8
|
|---|
| 66 | (p19) dep rem=Zlo,rem_4bitp,3,4 }
|
|---|
| 67 | { .mfi; (p19) xor Zhi=Zhi,Hhi
|
|---|
| 68 | ($p17) xor xi[1]=xi[1],in[1] };;
|
|---|
| 69 | { .mfi; (p18) ld8 Hhi=[Hi[1]]
|
|---|
| 70 | (p19) shrp Zlo=Zhi,Zlo,4 }
|
|---|
| 71 | { .mfi; (p19) ld8 rem=[rem]
|
|---|
| 72 | (p18) and Hi[1]=mask0xf0,xi[2] };;
|
|---|
| 73 | { .mmi; ($p16) ld1 in[0]=[inp],-1
|
|---|
| 74 | (p18) xor Zlo=Zlo,Hlo
|
|---|
| 75 | (p19) shr.u Zhi=Zhi,4 }
|
|---|
| 76 | { .mib; (p19) xor Hhi=Hhi,rem
|
|---|
| 77 | (p18) add Hi[1]=Htbl,Hi[1] };;
|
|---|
| 78 |
|
|---|
| 79 | { .mfi; (p18) ld8 Hlo=[Hi[1]],-8
|
|---|
| 80 | (p18) dep rem=Zlo,rem_4bitp,3,4 }
|
|---|
| 81 | { .mfi; (p17) shladd Hi[0]=xi[1],4,r0
|
|---|
| 82 | (p18) xor Zhi=Zhi,Hhi };;
|
|---|
| 83 | { .mfi; (p18) ld8 Hhi=[Hi[1]]
|
|---|
| 84 | (p18) shrp Zlo=Zhi,Zlo,4 }
|
|---|
| 85 | { .mfi; (p18) ld8 rem=[rem]
|
|---|
| 86 | (p17) and Hi[0]=mask0xf0,Hi[0] };;
|
|---|
| 87 | { .mmi; (p16) ld1 xi[0]=[Xi],-1
|
|---|
| 88 | (p18) xor Zlo=Zlo,Hlo
|
|---|
| 89 | (p18) shr.u Zhi=Zhi,4 }
|
|---|
| 90 | { .mib; (p18) xor Hhi=Hhi,rem
|
|---|
| 91 | (p17) add Hi[0]=Htbl,Hi[0]
|
|---|
| 92 | br.ctop.sptk $label };;
|
|---|
| 93 | ___
|
|---|
| 94 | }
|
|---|
| 95 |
|
|---|
| 96 | $code=<<___;
|
|---|
| 97 | .explicit
|
|---|
| 98 | .text
|
|---|
| 99 |
|
|---|
| 100 | prevfs=r2; prevlc=r3; prevpr=r8;
|
|---|
| 101 | mask0xf0=r21;
|
|---|
| 102 | rem=r22; rem_4bitp=r23;
|
|---|
| 103 | Xi=r24; Htbl=r25;
|
|---|
| 104 | inp=r26; end=r27;
|
|---|
| 105 | Hhi=r28; Hlo=r29;
|
|---|
| 106 | Zhi=r30; Zlo=r31;
|
|---|
| 107 |
|
|---|
| 108 | .align 128
|
|---|
| 109 | .skip 16 // aligns loop body
|
|---|
| 110 | .global gcm_gmult_4bit#
|
|---|
| 111 | .proc gcm_gmult_4bit#
|
|---|
| 112 | gcm_gmult_4bit:
|
|---|
| 113 | .prologue
|
|---|
| 114 | { .mmi; .save ar.pfs,prevfs
|
|---|
| 115 | alloc prevfs=ar.pfs,2,6,0,8
|
|---|
| 116 | $ADDP Xi=15,in0 // &Xi[15]
|
|---|
| 117 | mov rem_4bitp=ip }
|
|---|
| 118 | { .mii; $ADDP Htbl=8,in1 // &Htbl[0].lo
|
|---|
| 119 | .save ar.lc,prevlc
|
|---|
| 120 | mov prevlc=ar.lc
|
|---|
| 121 | .save pr,prevpr
|
|---|
| 122 | mov prevpr=pr };;
|
|---|
| 123 |
|
|---|
| 124 | .body
|
|---|
| 125 | .rotr in[3],xi[3],Hi[2]
|
|---|
| 126 |
|
|---|
| 127 | { .mib; ld1 xi[2]=[Xi],-1 // Xi[15]
|
|---|
| 128 | mov mask0xf0=0xf0
|
|---|
| 129 | brp.loop.imp .Loop1,.Lend1-16};;
|
|---|
| 130 | { .mmi; ld1 xi[1]=[Xi],-1 // Xi[14]
|
|---|
| 131 | };;
|
|---|
| 132 | { .mii; shladd Hi[1]=xi[2],4,r0
|
|---|
| 133 | mov pr.rot=0x7<<16
|
|---|
| 134 | mov ar.lc=13 };;
|
|---|
| 135 | { .mii; and Hi[1]=mask0xf0,Hi[1]
|
|---|
| 136 | mov ar.ec=3
|
|---|
| 137 | xor Zlo=Zlo,Zlo };;
|
|---|
| 138 | { .mii; add Hi[1]=Htbl,Hi[1] // &Htbl[nlo].lo
|
|---|
| 139 | add rem_4bitp=rem_4bit#-gcm_gmult_4bit#,rem_4bitp
|
|---|
| 140 | xor Zhi=Zhi,Zhi };;
|
|---|
| 141 | ___
|
|---|
| 142 | &loop (".Loop1",1);
|
|---|
| 143 | $code.=<<___;
|
|---|
| 144 | .Lend1:
|
|---|
| 145 | { .mib; xor Zhi=Zhi,Hhi };; // modulo-scheduling artefact
|
|---|
| 146 | { .mib; mux1 Zlo=Zlo,\@rev };;
|
|---|
| 147 | { .mib; mux1 Zhi=Zhi,\@rev };;
|
|---|
| 148 | { .mmi; add Hlo=9,Xi;; // ;; is here to prevent
|
|---|
| 149 | add Hhi=1,Xi };; // pipeline flush on Itanium
|
|---|
| 150 | { .mib; st8 [Hlo]=Zlo
|
|---|
| 151 | mov pr=prevpr,0x1ffff };;
|
|---|
| 152 | { .mib; st8 [Hhi]=Zhi
|
|---|
| 153 | mov ar.lc=prevlc
|
|---|
| 154 | br.ret.sptk.many b0 };;
|
|---|
| 155 | .endp gcm_gmult_4bit#
|
|---|
| 156 | ___
|
|---|
| 157 |
|
|---|
| 158 | ######################################################################
|
|---|
| 159 | # "528B" (well, "512B" actually) streamed GHASH
|
|---|
| 160 | #
|
|---|
| 161 | $Xip="in0";
|
|---|
| 162 | $Htbl="in1";
|
|---|
| 163 | $inp="in2";
|
|---|
| 164 | $len="in3";
|
|---|
| 165 | $rem_8bit="loc0";
|
|---|
| 166 | $mask0xff="loc1";
|
|---|
| 167 | ($sum,$rum) = $big_endian ? ("nop.m","nop.m") : ("sum","rum");
|
|---|
| 168 |
|
|---|
| 169 | sub load_htable() {
|
|---|
| 170 | for (my $i=0;$i<8;$i++) {
|
|---|
| 171 | $code.=<<___;
|
|---|
| 172 | { .mmi; ld8 r`16+2*$i+1`=[r8],16 // Htable[$i].hi
|
|---|
| 173 | ld8 r`16+2*$i`=[r9],16 } // Htable[$i].lo
|
|---|
| 174 | { .mmi; ldf8 f`32+2*$i+1`=[r10],16 // Htable[`8+$i`].hi
|
|---|
| 175 | ldf8 f`32+2*$i`=[r11],16 // Htable[`8+$i`].lo
|
|---|
| 176 | ___
|
|---|
| 177 | $code.=shift if (($i+$#_)==7);
|
|---|
| 178 | $code.="\t};;\n"
|
|---|
| 179 | }
|
|---|
| 180 | }
|
|---|
| 181 |
|
|---|
| 182 | $code.=<<___;
|
|---|
| 183 | prevsp=r3;
|
|---|
| 184 |
|
|---|
| 185 | .align 32
|
|---|
| 186 | .skip 16 // aligns loop body
|
|---|
| 187 | .global gcm_ghash_4bit#
|
|---|
| 188 | .proc gcm_ghash_4bit#
|
|---|
| 189 | gcm_ghash_4bit:
|
|---|
| 190 | .prologue
|
|---|
| 191 | { .mmi; .save ar.pfs,prevfs
|
|---|
| 192 | alloc prevfs=ar.pfs,4,2,0,0
|
|---|
| 193 | .vframe prevsp
|
|---|
| 194 | mov prevsp=sp
|
|---|
| 195 | mov $rem_8bit=ip };;
|
|---|
| 196 | .body
|
|---|
| 197 | { .mfi; $ADDP r8=0+0,$Htbl
|
|---|
| 198 | $ADDP r9=0+8,$Htbl }
|
|---|
| 199 | { .mfi; $ADDP r10=128+0,$Htbl
|
|---|
| 200 | $ADDP r11=128+8,$Htbl };;
|
|---|
| 201 | ___
|
|---|
| 202 | &load_htable(
|
|---|
| 203 | " $ADDP $Xip=15,$Xip", # &Xi[15]
|
|---|
| 204 | " $ADDP $len=$len,$inp", # &inp[len]
|
|---|
| 205 | " $ADDP $inp=15,$inp", # &inp[15]
|
|---|
| 206 | " mov $mask0xff=0xff",
|
|---|
| 207 | " add sp=-512,sp",
|
|---|
| 208 | " andcm sp=sp,$mask0xff", # align stack frame
|
|---|
| 209 | " add r14=0,sp",
|
|---|
| 210 | " add r15=8,sp");
|
|---|
| 211 | $code.=<<___;
|
|---|
| 212 | { .mmi; $sum 1<<1 // go big-endian
|
|---|
| 213 | add r8=256+0,sp
|
|---|
| 214 | add r9=256+8,sp }
|
|---|
| 215 | { .mmi; add r10=256+128+0,sp
|
|---|
| 216 | add r11=256+128+8,sp
|
|---|
| 217 | add $len=-17,$len };;
|
|---|
| 218 | ___
|
|---|
| 219 | for($i=0;$i<8;$i++) { # generate first half of Hshr4[]
|
|---|
| 220 | my ($rlo,$rhi)=("r".eval(16+2*$i),"r".eval(16+2*$i+1));
|
|---|
| 221 | $code.=<<___;
|
|---|
| 222 | { .mmi; st8 [r8]=$rlo,16 // Htable[$i].lo
|
|---|
| 223 | st8 [r9]=$rhi,16 // Htable[$i].hi
|
|---|
| 224 | shrp $rlo=$rhi,$rlo,4 }//;;
|
|---|
| 225 | { .mmi; stf8 [r10]=f`32+2*$i`,16 // Htable[`8+$i`].lo
|
|---|
| 226 | stf8 [r11]=f`32+2*$i+1`,16 // Htable[`8+$i`].hi
|
|---|
| 227 | shr.u $rhi=$rhi,4 };;
|
|---|
| 228 | { .mmi; st8 [r14]=$rlo,16 // Htable[$i].lo>>4
|
|---|
| 229 | st8 [r15]=$rhi,16 }//;; // Htable[$i].hi>>4
|
|---|
| 230 | ___
|
|---|
| 231 | }
|
|---|
| 232 | $code.=<<___;
|
|---|
| 233 | { .mmi; ld8 r16=[r8],16 // Htable[8].lo
|
|---|
| 234 | ld8 r17=[r9],16 };; // Htable[8].hi
|
|---|
| 235 | { .mmi; ld8 r18=[r8],16 // Htable[9].lo
|
|---|
| 236 | ld8 r19=[r9],16 } // Htable[9].hi
|
|---|
| 237 | { .mmi; rum 1<<5 // clear um.mfh
|
|---|
| 238 | shrp r16=r17,r16,4 };;
|
|---|
| 239 | ___
|
|---|
| 240 | for($i=0;$i<6;$i++) { # generate second half of Hshr4[]
|
|---|
| 241 | $code.=<<___;
|
|---|
| 242 | { .mmi; ld8 r`20+2*$i`=[r8],16 // Htable[`10+$i`].lo
|
|---|
| 243 | ld8 r`20+2*$i+1`=[r9],16 // Htable[`10+$i`].hi
|
|---|
| 244 | shr.u r`16+2*$i+1`=r`16+2*$i+1`,4 };;
|
|---|
| 245 | { .mmi; st8 [r14]=r`16+2*$i`,16 // Htable[`8+$i`].lo>>4
|
|---|
| 246 | st8 [r15]=r`16+2*$i+1`,16 // Htable[`8+$i`].hi>>4
|
|---|
| 247 | shrp r`18+2*$i`=r`18+2*$i+1`,r`18+2*$i`,4 }
|
|---|
| 248 | ___
|
|---|
| 249 | }
|
|---|
| 250 | $code.=<<___;
|
|---|
| 251 | { .mmi; shr.u r`16+2*$i+1`=r`16+2*$i+1`,4 };;
|
|---|
| 252 | { .mmi; st8 [r14]=r`16+2*$i`,16 // Htable[`8+$i`].lo>>4
|
|---|
| 253 | st8 [r15]=r`16+2*$i+1`,16 // Htable[`8+$i`].hi>>4
|
|---|
| 254 | shrp r`18+2*$i`=r`18+2*$i+1`,r`18+2*$i`,4 }
|
|---|
| 255 | { .mmi; add $Htbl=256,sp // &Htable[0]
|
|---|
| 256 | add $rem_8bit=rem_8bit#-gcm_ghash_4bit#,$rem_8bit
|
|---|
| 257 | shr.u r`18+2*$i+1`=r`18+2*$i+1`,4 };;
|
|---|
| 258 | { .mmi; st8 [r14]=r`18+2*$i` // Htable[`8+$i`].lo>>4
|
|---|
| 259 | st8 [r15]=r`18+2*$i+1` } // Htable[`8+$i`].hi>>4
|
|---|
| 260 | ___
|
|---|
| 261 |
|
|---|
| 262 | $in="r15";
|
|---|
| 263 | @xi=("r16","r17");
|
|---|
| 264 | @rem=("r18","r19");
|
|---|
| 265 | ($Alo,$Ahi,$Blo,$Bhi,$Zlo,$Zhi)=("r20","r21","r22","r23","r24","r25");
|
|---|
| 266 | ($Atbl,$Btbl)=("r26","r27");
|
|---|
| 267 |
|
|---|
| 268 | $code.=<<___; # (p16)
|
|---|
| 269 | { .mmi; ld1 $in=[$inp],-1 //(p16) *inp--
|
|---|
| 270 | ld1 $xi[0]=[$Xip],-1 //(p16) *Xi--
|
|---|
| 271 | cmp.eq p0,p6=r0,r0 };; // clear p6
|
|---|
| 272 | ___
|
|---|
| 273 | push (@xi,shift(@xi)); push (@rem,shift(@rem)); # "rotate" registers
|
|---|
| 274 |
|
|---|
| 275 | $code.=<<___; # (p16),(p17)
|
|---|
| 276 | { .mmi; ld1 $xi[0]=[$Xip],-1 //(p16) *Xi--
|
|---|
| 277 | xor $xi[1]=$xi[1],$in };; //(p17) xi=$xi[i]^inp[i]
|
|---|
| 278 | { .mii; ld1 $in=[$inp],-1 //(p16) *inp--
|
|---|
| 279 | dep $Atbl=$xi[1],$Htbl,4,4 //(p17) &Htable[nlo].lo
|
|---|
| 280 | and $xi[1]=-16,$xi[1] };; //(p17) nhi=xi&0xf0
|
|---|
| 281 | .align 32
|
|---|
| 282 | .LOOP:
|
|---|
| 283 | { .mmi;
|
|---|
| 284 | (p6) st8 [$Xip]=$Zhi,13
|
|---|
| 285 | xor $Zlo=$Zlo,$Zlo
|
|---|
| 286 | add $Btbl=$xi[1],$Htbl };; //(p17) &Htable[nhi].lo
|
|---|
| 287 | ___
|
|---|
| 288 | push (@xi,shift(@xi)); push (@rem,shift(@rem)); # "rotate" registers
|
|---|
| 289 |
|
|---|
| 290 | $code.=<<___; # (p16),(p17),(p18)
|
|---|
| 291 | { .mmi; ld8 $Alo=[$Atbl],8 //(p18) Htable[nlo].lo,&Htable[nlo].hi
|
|---|
| 292 | ld8 $rem[0]=[$Btbl],-256 //(p18) Htable[nhi].lo,&Hshr4[nhi].lo
|
|---|
| 293 | xor $xi[1]=$xi[1],$in };; //(p17) xi=$xi[i]^inp[i]
|
|---|
| 294 | { .mfi; ld8 $Ahi=[$Atbl] //(p18) Htable[nlo].hi
|
|---|
| 295 | dep $Atbl=$xi[1],$Htbl,4,4 } //(p17) &Htable[nlo].lo
|
|---|
| 296 | { .mfi; shladd $rem[0]=$rem[0],4,r0 //(p18) Htable[nhi].lo<<4
|
|---|
| 297 | xor $Zlo=$Zlo,$Alo };; //(p18) Z.lo^=Htable[nlo].lo
|
|---|
| 298 | { .mmi; ld8 $Blo=[$Btbl],8 //(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
|
|---|
| 299 | ld1 $in=[$inp],-1 } //(p16) *inp--
|
|---|
| 300 | { .mmi; xor $rem[0]=$rem[0],$Zlo //(p18) Z.lo^(Htable[nhi].lo<<4)
|
|---|
| 301 | mov $Zhi=$Ahi //(p18) Z.hi^=Htable[nlo].hi
|
|---|
| 302 | and $xi[1]=-16,$xi[1] };; //(p17) nhi=xi&0xf0
|
|---|
| 303 | { .mmi; ld8 $Bhi=[$Btbl] //(p18) Hshr4[nhi].hi
|
|---|
| 304 | ld1 $xi[0]=[$Xip],-1 //(p16) *Xi--
|
|---|
| 305 | shrp $Zlo=$Zhi,$Zlo,8 } //(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
|
|---|
| 306 | { .mmi; and $rem[0]=$rem[0],$mask0xff //(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
|
|---|
| 307 | add $Btbl=$xi[1],$Htbl };; //(p17) &Htable[nhi]
|
|---|
| 308 | ___
|
|---|
| 309 | push (@xi,shift(@xi)); push (@rem,shift(@rem)); # "rotate" registers
|
|---|
| 310 |
|
|---|
| 311 | for ($i=1;$i<14;$i++) {
|
|---|
| 312 | # Above and below fragments are derived from this one by removing
|
|---|
| 313 | # unsuitable (p??) instructions.
|
|---|
| 314 | $code.=<<___; # (p16),(p17),(p18),(p19)
|
|---|
| 315 | { .mmi; ld8 $Alo=[$Atbl],8 //(p18) Htable[nlo].lo,&Htable[nlo].hi
|
|---|
| 316 | ld8 $rem[0]=[$Btbl],-256 //(p18) Htable[nhi].lo,&Hshr4[nhi].lo
|
|---|
| 317 | shr.u $Zhi=$Zhi,8 } //(p19) Z.hi>>=8
|
|---|
| 318 | { .mmi; shladd $rem[1]=$rem[1],1,$rem_8bit //(p19) &rem_8bit[rem]
|
|---|
| 319 | xor $Zlo=$Zlo,$Blo //(p19) Z.lo^=Hshr4[nhi].lo
|
|---|
| 320 | xor $xi[1]=$xi[1],$in };; //(p17) xi=$xi[i]^inp[i]
|
|---|
| 321 | { .mmi; ld8 $Ahi=[$Atbl] //(p18) Htable[nlo].hi
|
|---|
| 322 | ld2 $rem[1]=[$rem[1]] //(p19) rem_8bit[rem]
|
|---|
| 323 | dep $Atbl=$xi[1],$Htbl,4,4 } //(p17) &Htable[nlo].lo
|
|---|
| 324 | { .mmi; shladd $rem[0]=$rem[0],4,r0 //(p18) Htable[nhi].lo<<4
|
|---|
| 325 | xor $Zlo=$Zlo,$Alo //(p18) Z.lo^=Htable[nlo].lo
|
|---|
| 326 | xor $Zhi=$Zhi,$Bhi };; //(p19) Z.hi^=Hshr4[nhi].hi
|
|---|
| 327 | { .mmi; ld8 $Blo=[$Btbl],8 //(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
|
|---|
| 328 | ld1 $in=[$inp],-1 //(p16) *inp--
|
|---|
| 329 | shl $rem[1]=$rem[1],48 } //(p19) rem_8bit[rem]<<48
|
|---|
| 330 | { .mmi; xor $rem[0]=$rem[0],$Zlo //(p18) Z.lo^(Htable[nhi].lo<<4)
|
|---|
| 331 | xor $Zhi=$Zhi,$Ahi //(p18) Z.hi^=Htable[nlo].hi
|
|---|
| 332 | and $xi[1]=-16,$xi[1] };; //(p17) nhi=xi&0xf0
|
|---|
| 333 | { .mmi; ld8 $Bhi=[$Btbl] //(p18) Hshr4[nhi].hi
|
|---|
| 334 | ld1 $xi[0]=[$Xip],-1 //(p16) *Xi--
|
|---|
| 335 | shrp $Zlo=$Zhi,$Zlo,8 } //(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
|
|---|
| 336 | { .mmi; and $rem[0]=$rem[0],$mask0xff //(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
|
|---|
| 337 | xor $Zhi=$Zhi,$rem[1] //(p19) Z.hi^=rem_8bit[rem]<<48
|
|---|
| 338 | add $Btbl=$xi[1],$Htbl };; //(p17) &Htable[nhi]
|
|---|
| 339 | ___
|
|---|
| 340 | push (@xi,shift(@xi)); push (@rem,shift(@rem)); # "rotate" registers
|
|---|
| 341 | }
|
|---|
| 342 |
|
|---|
| 343 | $code.=<<___; # (p17),(p18),(p19)
|
|---|
| 344 | { .mmi; ld8 $Alo=[$Atbl],8 //(p18) Htable[nlo].lo,&Htable[nlo].hi
|
|---|
| 345 | ld8 $rem[0]=[$Btbl],-256 //(p18) Htable[nhi].lo,&Hshr4[nhi].lo
|
|---|
| 346 | shr.u $Zhi=$Zhi,8 } //(p19) Z.hi>>=8
|
|---|
| 347 | { .mmi; shladd $rem[1]=$rem[1],1,$rem_8bit //(p19) &rem_8bit[rem]
|
|---|
| 348 | xor $Zlo=$Zlo,$Blo //(p19) Z.lo^=Hshr4[nhi].lo
|
|---|
| 349 | xor $xi[1]=$xi[1],$in };; //(p17) xi=$xi[i]^inp[i]
|
|---|
| 350 | { .mmi; ld8 $Ahi=[$Atbl] //(p18) Htable[nlo].hi
|
|---|
| 351 | ld2 $rem[1]=[$rem[1]] //(p19) rem_8bit[rem]
|
|---|
| 352 | dep $Atbl=$xi[1],$Htbl,4,4 };; //(p17) &Htable[nlo].lo
|
|---|
| 353 | { .mmi; shladd $rem[0]=$rem[0],4,r0 //(p18) Htable[nhi].lo<<4
|
|---|
| 354 | xor $Zlo=$Zlo,$Alo //(p18) Z.lo^=Htable[nlo].lo
|
|---|
| 355 | xor $Zhi=$Zhi,$Bhi };; //(p19) Z.hi^=Hshr4[nhi].hi
|
|---|
| 356 | { .mmi; ld8 $Blo=[$Btbl],8 //(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
|
|---|
| 357 | shl $rem[1]=$rem[1],48 } //(p19) rem_8bit[rem]<<48
|
|---|
| 358 | { .mmi; xor $rem[0]=$rem[0],$Zlo //(p18) Z.lo^(Htable[nhi].lo<<4)
|
|---|
| 359 | xor $Zhi=$Zhi,$Ahi //(p18) Z.hi^=Htable[nlo].hi
|
|---|
| 360 | and $xi[1]=-16,$xi[1] };; //(p17) nhi=xi&0xf0
|
|---|
| 361 | { .mmi; ld8 $Bhi=[$Btbl] //(p18) Hshr4[nhi].hi
|
|---|
| 362 | shrp $Zlo=$Zhi,$Zlo,8 } //(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
|
|---|
| 363 | { .mmi; and $rem[0]=$rem[0],$mask0xff //(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
|
|---|
| 364 | xor $Zhi=$Zhi,$rem[1] //(p19) Z.hi^=rem_8bit[rem]<<48
|
|---|
| 365 | add $Btbl=$xi[1],$Htbl };; //(p17) &Htable[nhi]
|
|---|
| 366 | ___
|
|---|
| 367 | push (@xi,shift(@xi)); push (@rem,shift(@rem)); # "rotate" registers
|
|---|
| 368 |
|
|---|
| 369 | $code.=<<___; # (p18),(p19)
|
|---|
| 370 | { .mfi; ld8 $Alo=[$Atbl],8 //(p18) Htable[nlo].lo,&Htable[nlo].hi
|
|---|
| 371 | shr.u $Zhi=$Zhi,8 } //(p19) Z.hi>>=8
|
|---|
| 372 | { .mfi; shladd $rem[1]=$rem[1],1,$rem_8bit //(p19) &rem_8bit[rem]
|
|---|
| 373 | xor $Zlo=$Zlo,$Blo };; //(p19) Z.lo^=Hshr4[nhi].lo
|
|---|
| 374 | { .mfi; ld8 $Ahi=[$Atbl] //(p18) Htable[nlo].hi
|
|---|
| 375 | xor $Zlo=$Zlo,$Alo } //(p18) Z.lo^=Htable[nlo].lo
|
|---|
| 376 | { .mfi; ld2 $rem[1]=[$rem[1]] //(p19) rem_8bit[rem]
|
|---|
| 377 | xor $Zhi=$Zhi,$Bhi };; //(p19) Z.hi^=Hshr4[nhi].hi
|
|---|
| 378 | { .mfi; ld8 $Blo=[$Btbl],8 //(p18) Htable[nhi].lo,&Htable[nhi].hi
|
|---|
| 379 | shl $rem[1]=$rem[1],48 } //(p19) rem_8bit[rem]<<48
|
|---|
| 380 | { .mfi; shladd $rem[0]=$Zlo,4,r0 //(p18) Z.lo<<4
|
|---|
| 381 | xor $Zhi=$Zhi,$Ahi };; //(p18) Z.hi^=Htable[nlo].hi
|
|---|
| 382 | { .mfi; ld8 $Bhi=[$Btbl] //(p18) Htable[nhi].hi
|
|---|
| 383 | shrp $Zlo=$Zhi,$Zlo,4 } //(p18) Z.lo=(Z.hi<<60)|(Z.lo>>4)
|
|---|
| 384 | { .mfi; and $rem[0]=$rem[0],$mask0xff //(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
|
|---|
| 385 | xor $Zhi=$Zhi,$rem[1] };; //(p19) Z.hi^=rem_8bit[rem]<<48
|
|---|
| 386 | ___
|
|---|
| 387 | push (@xi,shift(@xi)); push (@rem,shift(@rem)); # "rotate" registers
|
|---|
| 388 |
|
|---|
| 389 | $code.=<<___; # (p19)
|
|---|
| 390 | { .mmi; cmp.ltu p6,p0=$inp,$len
|
|---|
| 391 | add $inp=32,$inp
|
|---|
| 392 | shr.u $Zhi=$Zhi,4 } //(p19) Z.hi>>=4
|
|---|
| 393 | { .mmi; shladd $rem[1]=$rem[1],1,$rem_8bit //(p19) &rem_8bit[rem]
|
|---|
| 394 | xor $Zlo=$Zlo,$Blo //(p19) Z.lo^=Hshr4[nhi].lo
|
|---|
| 395 | add $Xip=9,$Xip };; // &Xi.lo
|
|---|
| 396 | { .mmi; ld2 $rem[1]=[$rem[1]] //(p19) rem_8bit[rem]
|
|---|
| 397 | (p6) ld1 $in=[$inp],-1 //[p16] *inp--
|
|---|
| 398 | (p6) extr.u $xi[1]=$Zlo,8,8 } //[p17] Xi[14]
|
|---|
| 399 | { .mmi; xor $Zhi=$Zhi,$Bhi //(p19) Z.hi^=Hshr4[nhi].hi
|
|---|
| 400 | (p6) and $xi[0]=$Zlo,$mask0xff };; //[p16] Xi[15]
|
|---|
| 401 | { .mmi; st8 [$Xip]=$Zlo,-8
|
|---|
| 402 | (p6) xor $xi[0]=$xi[0],$in //[p17] xi=$xi[i]^inp[i]
|
|---|
| 403 | shl $rem[1]=$rem[1],48 };; //(p19) rem_8bit[rem]<<48
|
|---|
| 404 | { .mmi;
|
|---|
| 405 | (p6) ld1 $in=[$inp],-1 //[p16] *inp--
|
|---|
| 406 | xor $Zhi=$Zhi,$rem[1] //(p19) Z.hi^=rem_8bit[rem]<<48
|
|---|
| 407 | (p6) dep $Atbl=$xi[0],$Htbl,4,4 } //[p17] &Htable[nlo].lo
|
|---|
| 408 | { .mib;
|
|---|
| 409 | (p6) and $xi[0]=-16,$xi[0] //[p17] nhi=xi&0xf0
|
|---|
| 410 | (p6) br.cond.dptk.many .LOOP };;
|
|---|
| 411 |
|
|---|
| 412 | { .mib; st8 [$Xip]=$Zhi };;
|
|---|
| 413 | { .mib; $rum 1<<1 // return to little-endian
|
|---|
| 414 | .restore sp
|
|---|
| 415 | mov sp=prevsp
|
|---|
| 416 | br.ret.sptk.many b0 };;
|
|---|
| 417 | .endp gcm_ghash_4bit#
|
|---|
| 418 | ___
|
|---|
| 419 | $code.=<<___;
|
|---|
| 420 | .align 128
|
|---|
| 421 | .type rem_4bit#,\@object
|
|---|
| 422 | rem_4bit:
|
|---|
| 423 | data8 0x0000<<48, 0x1C20<<48, 0x3840<<48, 0x2460<<48
|
|---|
| 424 | data8 0x7080<<48, 0x6CA0<<48, 0x48C0<<48, 0x54E0<<48
|
|---|
| 425 | data8 0xE100<<48, 0xFD20<<48, 0xD940<<48, 0xC560<<48
|
|---|
| 426 | data8 0x9180<<48, 0x8DA0<<48, 0xA9C0<<48, 0xB5E0<<48
|
|---|
| 427 | .size rem_4bit#,128
|
|---|
| 428 | .type rem_8bit#,\@object
|
|---|
| 429 | rem_8bit:
|
|---|
| 430 | data1 0x00,0x00, 0x01,0xC2, 0x03,0x84, 0x02,0x46, 0x07,0x08, 0x06,0xCA, 0x04,0x8C, 0x05,0x4E
|
|---|
| 431 | data1 0x0E,0x10, 0x0F,0xD2, 0x0D,0x94, 0x0C,0x56, 0x09,0x18, 0x08,0xDA, 0x0A,0x9C, 0x0B,0x5E
|
|---|
| 432 | data1 0x1C,0x20, 0x1D,0xE2, 0x1F,0xA4, 0x1E,0x66, 0x1B,0x28, 0x1A,0xEA, 0x18,0xAC, 0x19,0x6E
|
|---|
| 433 | data1 0x12,0x30, 0x13,0xF2, 0x11,0xB4, 0x10,0x76, 0x15,0x38, 0x14,0xFA, 0x16,0xBC, 0x17,0x7E
|
|---|
| 434 | data1 0x38,0x40, 0x39,0x82, 0x3B,0xC4, 0x3A,0x06, 0x3F,0x48, 0x3E,0x8A, 0x3C,0xCC, 0x3D,0x0E
|
|---|
| 435 | data1 0x36,0x50, 0x37,0x92, 0x35,0xD4, 0x34,0x16, 0x31,0x58, 0x30,0x9A, 0x32,0xDC, 0x33,0x1E
|
|---|
| 436 | data1 0x24,0x60, 0x25,0xA2, 0x27,0xE4, 0x26,0x26, 0x23,0x68, 0x22,0xAA, 0x20,0xEC, 0x21,0x2E
|
|---|
| 437 | data1 0x2A,0x70, 0x2B,0xB2, 0x29,0xF4, 0x28,0x36, 0x2D,0x78, 0x2C,0xBA, 0x2E,0xFC, 0x2F,0x3E
|
|---|
| 438 | data1 0x70,0x80, 0x71,0x42, 0x73,0x04, 0x72,0xC6, 0x77,0x88, 0x76,0x4A, 0x74,0x0C, 0x75,0xCE
|
|---|
| 439 | data1 0x7E,0x90, 0x7F,0x52, 0x7D,0x14, 0x7C,0xD6, 0x79,0x98, 0x78,0x5A, 0x7A,0x1C, 0x7B,0xDE
|
|---|
| 440 | data1 0x6C,0xA0, 0x6D,0x62, 0x6F,0x24, 0x6E,0xE6, 0x6B,0xA8, 0x6A,0x6A, 0x68,0x2C, 0x69,0xEE
|
|---|
| 441 | data1 0x62,0xB0, 0x63,0x72, 0x61,0x34, 0x60,0xF6, 0x65,0xB8, 0x64,0x7A, 0x66,0x3C, 0x67,0xFE
|
|---|
| 442 | data1 0x48,0xC0, 0x49,0x02, 0x4B,0x44, 0x4A,0x86, 0x4F,0xC8, 0x4E,0x0A, 0x4C,0x4C, 0x4D,0x8E
|
|---|
| 443 | data1 0x46,0xD0, 0x47,0x12, 0x45,0x54, 0x44,0x96, 0x41,0xD8, 0x40,0x1A, 0x42,0x5C, 0x43,0x9E
|
|---|
| 444 | data1 0x54,0xE0, 0x55,0x22, 0x57,0x64, 0x56,0xA6, 0x53,0xE8, 0x52,0x2A, 0x50,0x6C, 0x51,0xAE
|
|---|
| 445 | data1 0x5A,0xF0, 0x5B,0x32, 0x59,0x74, 0x58,0xB6, 0x5D,0xF8, 0x5C,0x3A, 0x5E,0x7C, 0x5F,0xBE
|
|---|
| 446 | data1 0xE1,0x00, 0xE0,0xC2, 0xE2,0x84, 0xE3,0x46, 0xE6,0x08, 0xE7,0xCA, 0xE5,0x8C, 0xE4,0x4E
|
|---|
| 447 | data1 0xEF,0x10, 0xEE,0xD2, 0xEC,0x94, 0xED,0x56, 0xE8,0x18, 0xE9,0xDA, 0xEB,0x9C, 0xEA,0x5E
|
|---|
| 448 | data1 0xFD,0x20, 0xFC,0xE2, 0xFE,0xA4, 0xFF,0x66, 0xFA,0x28, 0xFB,0xEA, 0xF9,0xAC, 0xF8,0x6E
|
|---|
| 449 | data1 0xF3,0x30, 0xF2,0xF2, 0xF0,0xB4, 0xF1,0x76, 0xF4,0x38, 0xF5,0xFA, 0xF7,0xBC, 0xF6,0x7E
|
|---|
| 450 | data1 0xD9,0x40, 0xD8,0x82, 0xDA,0xC4, 0xDB,0x06, 0xDE,0x48, 0xDF,0x8A, 0xDD,0xCC, 0xDC,0x0E
|
|---|
| 451 | data1 0xD7,0x50, 0xD6,0x92, 0xD4,0xD4, 0xD5,0x16, 0xD0,0x58, 0xD1,0x9A, 0xD3,0xDC, 0xD2,0x1E
|
|---|
| 452 | data1 0xC5,0x60, 0xC4,0xA2, 0xC6,0xE4, 0xC7,0x26, 0xC2,0x68, 0xC3,0xAA, 0xC1,0xEC, 0xC0,0x2E
|
|---|
| 453 | data1 0xCB,0x70, 0xCA,0xB2, 0xC8,0xF4, 0xC9,0x36, 0xCC,0x78, 0xCD,0xBA, 0xCF,0xFC, 0xCE,0x3E
|
|---|
| 454 | data1 0x91,0x80, 0x90,0x42, 0x92,0x04, 0x93,0xC6, 0x96,0x88, 0x97,0x4A, 0x95,0x0C, 0x94,0xCE
|
|---|
| 455 | data1 0x9F,0x90, 0x9E,0x52, 0x9C,0x14, 0x9D,0xD6, 0x98,0x98, 0x99,0x5A, 0x9B,0x1C, 0x9A,0xDE
|
|---|
| 456 | data1 0x8D,0xA0, 0x8C,0x62, 0x8E,0x24, 0x8F,0xE6, 0x8A,0xA8, 0x8B,0x6A, 0x89,0x2C, 0x88,0xEE
|
|---|
| 457 | data1 0x83,0xB0, 0x82,0x72, 0x80,0x34, 0x81,0xF6, 0x84,0xB8, 0x85,0x7A, 0x87,0x3C, 0x86,0xFE
|
|---|
| 458 | data1 0xA9,0xC0, 0xA8,0x02, 0xAA,0x44, 0xAB,0x86, 0xAE,0xC8, 0xAF,0x0A, 0xAD,0x4C, 0xAC,0x8E
|
|---|
| 459 | data1 0xA7,0xD0, 0xA6,0x12, 0xA4,0x54, 0xA5,0x96, 0xA0,0xD8, 0xA1,0x1A, 0xA3,0x5C, 0xA2,0x9E
|
|---|
| 460 | data1 0xB5,0xE0, 0xB4,0x22, 0xB6,0x64, 0xB7,0xA6, 0xB2,0xE8, 0xB3,0x2A, 0xB1,0x6C, 0xB0,0xAE
|
|---|
| 461 | data1 0xBB,0xF0, 0xBA,0x32, 0xB8,0x74, 0xB9,0xB6, 0xBC,0xF8, 0xBD,0x3A, 0xBF,0x7C, 0xBE,0xBE
|
|---|
| 462 | .size rem_8bit#,512
|
|---|
| 463 | stringz "GHASH for IA64, CRYPTOGAMS by <appro\@openssl.org>"
|
|---|
| 464 | ___
|
|---|
| 465 |
|
|---|
| 466 | $code =~ s/mux1(\s+)\S+\@rev/nop.i$1 0x0/gm if ($big_endian);
|
|---|
| 467 | $code =~ s/\`([^\`]*)\`/eval $1/gem;
|
|---|
| 468 |
|
|---|
| 469 | print $code;
|
|---|
| 470 | close STDOUT or die "error closing STDOUT: $!";
|
|---|
