?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
OpenSSL: update to 1.0.2a
[tomato.git] / release / src / router / openssl / crypto / sha / asm / sha512-armv8.pl
blobbd7a0a5662e0a236f8f35dcdedd83954c9a7d1f8
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # SHA256/512 for ARMv8.
11 #
12 # Performance in cycles per processed byte and improvement coefficient
13 # over code generated with "default" compiler:
14 #
15 # SHA256-hw SHA256(*) SHA512
16 # Apple A7 1.97 10.5 (+33%) 6.73 (-1%(**))
17 # Cortex-A53 2.38 15.6 (+110%) 10.1 (+190%(***))
18 # Cortex-A57 2.31 11.6 (+86%) 7.51 (+260%(***))
19 #
20 # (*) Software SHA256 results are of lesser relevance, presented
21 # mostly for informational purposes.
22 # (**) The result is a trade-off: it's possible to improve it by
23 # 10% (or by 1 cycle per round), but at the cost of 20% loss
24 # on Cortex-A53 (or by 4 cycles per round).
25 # (***) Super-impressive coefficients over gcc-generated code are
26 # indication of some compiler "pathology", most notably code
27 # generated with -mgeneral-regs-only is significanty faster
28 # and lags behind assembly only by 50-90%.
29
30 $flavour=shift;
31 $output=shift;
32 open STDOUT,">$output";
33
34 if ($output =~ /512/) {
35 $BITS=512;
36 $SZ=8;
37 @Sigma0=(28,34,39);
38 @Sigma1=(14,18,41);
39 @sigma0=(1, 8, 7);
40 @sigma1=(19,61, 6);
41 $rounds=80;
42 $reg_t="x";
43 } else {
44 $BITS=256;
45 $SZ=4;
46 @Sigma0=( 2,13,22);
47 @Sigma1=( 6,11,25);
48 @sigma0=( 7,18, 3);
49 @sigma1=(17,19,10);
50 $rounds=64;
51 $reg_t="w";
52 }
53
54 $func="sha${BITS}_block_data_order";
55
56 ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
57
58 @X=map("$reg_t$_",(3..15,0..2));
59 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
60 ($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
61
62 sub BODY_00_xx {
63 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
64 my $j=($i+1)&15;
65 my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
66 $T0=@X[$i+3] if ($i<11);
67
68 $code.=<<___ if ($i<16);
69 #ifndef __ARMEB__
70 rev @X[$i],@X[$i] // $i
71 #endif
72 ___
73 $code.=<<___ if ($i<13 && ($i&1));
74 ldp @X[$i+1],@X[$i+2],[$inp],#2*$SZ
75 ___
76 $code.=<<___ if ($i==13);
77 ldp @X[14],@X[15],[$inp]
78 ___
79 $code.=<<___ if ($i>=14);
80 ldr @X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
81 ___
82 $code.=<<___ if ($i>0 && $i<16);
83 add $a,$a,$t1 // h+=Sigma0(a)
84 ___
85 $code.=<<___ if ($i>=11);
86 str @X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
87 ___
88 # While ARMv8 specifies merged rotate-n-logical operation such as
89 # 'eor x,y,z,ror#n', it was found to negatively affect performance
90 # on Apple A7. The reason seems to be that it requires even 'y' to
91 # be available earlier. This means that such merged instruction is
92 # not necessarily best choice on critical path... On the other hand
93 # Cortex-A5x handles merged instructions much better than disjoint
94 # rotate and logical... See (**) footnote above.
95 $code.=<<___ if ($i<15);
96 ror $t0,$e,#$Sigma1[0]
97 add $h,$h,$t2 // h+=K[i]
98 eor $T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
99 and $t1,$f,$e
100 bic $t2,$g,$e
101 add $h,$h,@X[$i&15] // h+=X[i]
102 orr $t1,$t1,$t2 // Ch(e,f,g)
103 eor $t2,$a,$b // a^b, b^c in next round
104 eor $t0,$t0,$T0,ror#$Sigma1[1] // Sigma1(e)
105 ror $T0,$a,#$Sigma0[0]
106 add $h,$h,$t1 // h+=Ch(e,f,g)
107 eor $t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
108 add $h,$h,$t0 // h+=Sigma1(e)
109 and $t3,$t3,$t2 // (b^c)&=(a^b)
110 add $d,$d,$h // d+=h
111 eor $t3,$t3,$b // Maj(a,b,c)
112 eor $t1,$T0,$t1,ror#$Sigma0[1] // Sigma0(a)
113 add $h,$h,$t3 // h+=Maj(a,b,c)
114 ldr $t3,[$Ktbl],#$SZ // *K++, $t2 in next round
115 //add $h,$h,$t1 // h+=Sigma0(a)
116 ___
117 $code.=<<___ if ($i>=15);
118 ror $t0,$e,#$Sigma1[0]
119 add $h,$h,$t2 // h+=K[i]
120 ror $T1,@X[($j+1)&15],#$sigma0[0]
121 and $t1,$f,$e
122 ror $T2,@X[($j+14)&15],#$sigma1[0]
123 bic $t2,$g,$e
124 ror $T0,$a,#$Sigma0[0]
125 add $h,$h,@X[$i&15] // h+=X[i]
126 eor $t0,$t0,$e,ror#$Sigma1[1]
127 eor $T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
128 orr $t1,$t1,$t2 // Ch(e,f,g)
129 eor $t2,$a,$b // a^b, b^c in next round
130 eor $t0,$t0,$e,ror#$Sigma1[2] // Sigma1(e)
131 eor $T0,$T0,$a,ror#$Sigma0[1]
132 add $h,$h,$t1 // h+=Ch(e,f,g)
133 and $t3,$t3,$t2 // (b^c)&=(a^b)
134 eor $T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
135 eor $T1,$T1,@X[($j+1)&15],lsr#$sigma0[2] // sigma0(X[i+1])
136 add $h,$h,$t0 // h+=Sigma1(e)
137 eor $t3,$t3,$b // Maj(a,b,c)
138 eor $t1,$T0,$a,ror#$Sigma0[2] // Sigma0(a)
139 eor $T2,$T2,@X[($j+14)&15],lsr#$sigma1[2] // sigma1(X[i+14])
140 add @X[$j],@X[$j],@X[($j+9)&15]
141 add $d,$d,$h // d+=h
142 add $h,$h,$t3 // h+=Maj(a,b,c)
143 ldr $t3,[$Ktbl],#$SZ // *K++, $t2 in next round
144 add @X[$j],@X[$j],$T1
145 add $h,$h,$t1 // h+=Sigma0(a)
146 add @X[$j],@X[$j],$T2
147 ___
148 ($t2,$t3)=($t3,$t2);
149 }
150
151 $code.=<<___;
152 #include "arm_arch.h"
153
154 .text
155
156 .globl $func
157 .type $func,%function
158 .align 6
159 $func:
160 ___
161 $code.=<<___ if ($SZ==4);
162 ldr x16,.LOPENSSL_armcap_P
163 adr x17,.LOPENSSL_armcap_P
164 add x16,x16,x17
165 ldr w16,[x16]
166 tst w16,#ARMV8_SHA256
167 b.ne .Lv8_entry
168 ___
169 $code.=<<___;
170 stp x29,x30,[sp,#-128]!
171 add x29,sp,#0
172
173 stp x19,x20,[sp,#16]
174 stp x21,x22,[sp,#32]
175 stp x23,x24,[sp,#48]
176 stp x25,x26,[sp,#64]
177 stp x27,x28,[sp,#80]
178 sub sp,sp,#4*$SZ
179
180 ldp $A,$B,[$ctx] // load context
181 ldp $C,$D,[$ctx,#2*$SZ]
182 ldp $E,$F,[$ctx,#4*$SZ]
183 add $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input
184 ldp $G,$H,[$ctx,#6*$SZ]
185 adr $Ktbl,K$BITS
186 stp $ctx,$num,[x29,#96]
187
188 .Loop:
189 ldp @X[0],@X[1],[$inp],#2*$SZ
190 ldr $t2,[$Ktbl],#$SZ // *K++
191 eor $t3,$B,$C // magic seed
192 str $inp,[x29,#112]
193 ___
194 for ($i=0;$i<16;$i++) { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
195 $code.=".Loop_16_xx:\n";
196 for (;$i<32;$i++) { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
197 $code.=<<___;
198 cbnz $t2,.Loop_16_xx
199
200 ldp $ctx,$num,[x29,#96]
201 ldr $inp,[x29,#112]
202 sub $Ktbl,$Ktbl,#`$SZ*($rounds+1)` // rewind
203
204 ldp @X[0],@X[1],[$ctx]
205 ldp @X[2],@X[3],[$ctx,#2*$SZ]
206 add $inp,$inp,#14*$SZ // advance input pointer
207 ldp @X[4],@X[5],[$ctx,#4*$SZ]
208 add $A,$A,@X[0]
209 ldp @X[6],@X[7],[$ctx,#6*$SZ]
210 add $B,$B,@X[1]
211 add $C,$C,@X[2]
212 add $D,$D,@X[3]
213 stp $A,$B,[$ctx]
214 add $E,$E,@X[4]
215 add $F,$F,@X[5]
216 stp $C,$D,[$ctx,#2*$SZ]
217 add $G,$G,@X[6]
218 add $H,$H,@X[7]
219 cmp $inp,$num
220 stp $E,$F,[$ctx,#4*$SZ]
221 stp $G,$H,[$ctx,#6*$SZ]
222 b.ne .Loop
223
224 ldp x19,x20,[x29,#16]
225 add sp,sp,#4*$SZ
226 ldp x21,x22,[x29,#32]
227 ldp x23,x24,[x29,#48]
228 ldp x25,x26,[x29,#64]
229 ldp x27,x28,[x29,#80]
230 ldp x29,x30,[sp],#128
231 ret
232 .size $func,.-$func
233
234 .align 6
235 .type K$BITS,%object
236 K$BITS:
237 ___
238 $code.=<<___ if ($SZ==8);
239 .quad 0x428a2f98d728ae22,0x7137449123ef65cd
240 .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
241 .quad 0x3956c25bf348b538,0x59f111f1b605d019
242 .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
243 .quad 0xd807aa98a3030242,0x12835b0145706fbe
244 .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
245 .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
246 .quad 0x9bdc06a725c71235,0xc19bf174cf692694
247 .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
248 .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
249 .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
250 .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
251 .quad 0x983e5152ee66dfab,0xa831c66d2db43210
252 .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
253 .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
254 .quad 0x06ca6351e003826f,0x142929670a0e6e70
255 .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
256 .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
257 .quad 0x650a73548baf63de,0x766a0abb3c77b2a8
258 .quad 0x81c2c92e47edaee6,0x92722c851482353b
259 .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
260 .quad 0xc24b8b70d0f89791,0xc76c51a30654be30
261 .quad 0xd192e819d6ef5218,0xd69906245565a910
262 .quad 0xf40e35855771202a,0x106aa07032bbd1b8
263 .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
264 .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
265 .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
266 .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
267 .quad 0x748f82ee5defb2fc,0x78a5636f43172f60
268 .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
269 .quad 0x90befffa23631e28,0xa4506cebde82bde9
270 .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
271 .quad 0xca273eceea26619c,0xd186b8c721c0c207
272 .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
273 .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
274 .quad 0x113f9804bef90dae,0x1b710b35131c471b
275 .quad 0x28db77f523047d84,0x32caab7b40c72493
276 .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
277 .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
278 .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
279 .quad 0 // terminator
280 ___
281 $code.=<<___ if ($SZ==4);
282 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
283 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
284 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
285 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
286 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
287 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
288 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
289 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
290 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
291 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
292 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
293 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
294 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
295 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
296 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
297 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
298 .long 0 //terminator
299 ___
300 $code.=<<___;
301 .size K$BITS,.-K$BITS
302 .align 3
303 .LOPENSSL_armcap_P:
304 .quad OPENSSL_armcap_P-.
305 .asciz "SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
306 .align 2
307 ___
308
309 if ($SZ==4) {
310 my $Ktbl="x3";
311
312 my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
313 my @MSG=map("v$_.16b",(4..7));
314 my ($W0,$W1)=("v16.4s","v17.4s");
315 my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
316
317 $code.=<<___;
318 .type sha256_block_armv8,%function
319 .align 6
320 sha256_block_armv8:
321 .Lv8_entry:
322 stp x29,x30,[sp,#-16]!
323 add x29,sp,#0
324
325 ld1.32 {$ABCD,$EFGH},[$ctx]
326 adr $Ktbl,K256
327
328 .Loop_hw:
329 ld1 {@MSG[0]-@MSG[3]},[$inp],#64
330 sub $num,$num,#1
331 ld1.32 {$W0},[$Ktbl],#16
332 rev32 @MSG[0],@MSG[0]
333 rev32 @MSG[1],@MSG[1]
334 rev32 @MSG[2],@MSG[2]
335 rev32 @MSG[3],@MSG[3]
336 orr $ABCD_SAVE,$ABCD,$ABCD // offload
337 orr $EFGH_SAVE,$EFGH,$EFGH
338 ___
339 for($i=0;$i<12;$i++) {
340 $code.=<<___;
341 ld1.32 {$W1},[$Ktbl],#16
342 add.i32 $W0,$W0,@MSG[0]
343 sha256su0 @MSG[0],@MSG[1]
344 orr $abcd,$ABCD,$ABCD
345 sha256h $ABCD,$EFGH,$W0
346 sha256h2 $EFGH,$abcd,$W0
347 sha256su1 @MSG[0],@MSG[2],@MSG[3]
348 ___
349 ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
350 }
351 $code.=<<___;
352 ld1.32 {$W1},[$Ktbl],#16
353 add.i32 $W0,$W0,@MSG[0]
354 orr $abcd,$ABCD,$ABCD
355 sha256h $ABCD,$EFGH,$W0
356 sha256h2 $EFGH,$abcd,$W0
357
358 ld1.32 {$W0},[$Ktbl],#16
359 add.i32 $W1,$W1,@MSG[1]
360 orr $abcd,$ABCD,$ABCD
361 sha256h $ABCD,$EFGH,$W1
362 sha256h2 $EFGH,$abcd,$W1
363
364 ld1.32 {$W1},[$Ktbl]
365 add.i32 $W0,$W0,@MSG[2]
366 sub $Ktbl,$Ktbl,#$rounds*$SZ-16 // rewind
367 orr $abcd,$ABCD,$ABCD
368 sha256h $ABCD,$EFGH,$W0
369 sha256h2 $EFGH,$abcd,$W0
370
371 add.i32 $W1,$W1,@MSG[3]
372 orr $abcd,$ABCD,$ABCD
373 sha256h $ABCD,$EFGH,$W1
374 sha256h2 $EFGH,$abcd,$W1
375
376 add.i32 $ABCD,$ABCD,$ABCD_SAVE
377 add.i32 $EFGH,$EFGH,$EFGH_SAVE
378
379 cbnz $num,.Loop_hw
380
381 st1.32 {$ABCD,$EFGH},[$ctx]
382
383 ldr x29,[sp],#16
384 ret
385 .size sha256_block_armv8,.-sha256_block_armv8
386 ___
387 }
388
389 $code.=<<___;
390 .comm OPENSSL_armcap_P,4,4
391 ___
392
393 { my %opcode = (
394 "sha256h" => 0x5e004000, "sha256h2" => 0x5e005000,
395 "sha256su0" => 0x5e282800, "sha256su1" => 0x5e006000 );
396
397 sub unsha256 {
398 my ($mnemonic,$arg)=@_;
399
400 $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
401 &&
402 sprintf ".inst\t0x%08x\t//%s %s",
403 $opcode{$mnemonic}|$1|($2<<5)|($3<<16),
404 $mnemonic,$arg;
405 }
406 }
407
408 foreach(split("\n",$code)) {
409
410 s/\`([^\`]*)\`/eval($1)/geo;
411
412 s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/geo;
413
414 s/\.\w?32\b//o and s/\.16b/\.4s/go;
415 m/(ld|st)1[^\[]+\[0\]/o and s/\.4s/\.s/go;
416
417 print $_,"\n";
418 }
419
420 close STDOUT;
Tomato firmware and modifications.