?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
import libcrypto (LibreSSL 2.5.2)
[unleashed.git] / lib / libcrypto / bn / asm / modexp512-x86_64.pl
blob4317282835385903efc14e977366d3a988389b4b
1 #!/usr/bin/env perl
2 #
3 # Copyright (c) 2010-2011 Intel Corp.
4 # Author: Vinodh.Gopal@intel.com
5 # Jim Guilford
6 # Erdinc.Ozturk@intel.com
7 # Maxim.Perminov@intel.com
8 #
9 # More information about algorithm used can be found at:
10 # http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf
11 #
12 # ====================================================================
13 # Copyright (c) 2011 The OpenSSL Project. All rights reserved.
14 #
15 # Redistribution and use in source and binary forms, with or without
16 # modification, are permitted provided that the following conditions
17 # are met:
18 #
19 # 1. Redistributions of source code must retain the above copyright
20 # notice, this list of conditions and the following disclaimer.
21 #
22 # 2. Redistributions in binary form must reproduce the above copyright
23 # notice, this list of conditions and the following disclaimer in
24 # the documentation and/or other materials provided with the
25 # distribution.
26 #
27 # 3. All advertising materials mentioning features or use of this
28 # software must display the following acknowledgment:
29 # "This product includes software developed by the OpenSSL Project
30 # for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
31 #
32 # 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
33 # endorse or promote products derived from this software without
34 # prior written permission. For written permission, please contact
35 # licensing@OpenSSL.org.
36 #
37 # 5. Products derived from this software may not be called "OpenSSL"
38 # nor may "OpenSSL" appear in their names without prior written
39 # permission of the OpenSSL Project.
40 #
41 # 6. Redistributions of any form whatsoever must retain the following
42 # acknowledgment:
43 # "This product includes software developed by the OpenSSL Project
44 # for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
45 #
46 # THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
47 # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
49 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
50 # ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
51 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
52 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
53 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
54 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
55 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
56 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
57 # OF THE POSSIBILITY OF SUCH DAMAGE.
58 # ====================================================================
59
60 $flavour = shift;
61 $output = shift;
62 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
63
64 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
65 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
66 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
67 die "can't locate x86_64-xlate.pl";
68
69 open OUT,"| \"$^X\" $xlate $flavour $output";
70 *STDOUT=*OUT;
71
72 use strict;
73 my $code=".text\n\n";
74 my $m=0;
75
76 #
77 # Define x512 macros
78 #
79
80 #MULSTEP_512_ADD MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src1, src2, add_src, tmp1, tmp2
81 #
82 # uses rax, rdx, and args
83 sub MULSTEP_512_ADD
84 {
85 my ($x, $DST, $SRC2, $ASRC, $OP, $TMP)=@_;
86 my @X=@$x; # make a copy
87 $code.=<<___;
88 mov (+8*0)($SRC2), %rax
89 mul $OP # rdx:rax = %OP * [0]
90 mov ($ASRC), $X[0]
91 add %rax, $X[0]
92 adc \$0, %rdx
93 mov $X[0], $DST
94 ___
95 for(my $i=1;$i<8;$i++) {
96 $code.=<<___;
97 mov %rdx, $TMP
98
99 mov (+8*$i)($SRC2), %rax
100 mul $OP # rdx:rax = %OP * [$i]
101 mov (+8*$i)($ASRC), $X[$i]
102 add %rax, $X[$i]
103 adc \$0, %rdx
104 add $TMP, $X[$i]
105 adc \$0, %rdx
106 ___
107 }
108 $code.=<<___;
109 mov %rdx, $X[0]
110 ___
111 }
112
113 #MULSTEP_512 MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src2, src1_val, tmp
114 #
115 # uses rax, rdx, and args
116 sub MULSTEP_512
117 {
118 my ($x, $DST, $SRC2, $OP, $TMP)=@_;
119 my @X=@$x; # make a copy
120 $code.=<<___;
121 mov (+8*0)($SRC2), %rax
122 mul $OP # rdx:rax = %OP * [0]
123 add %rax, $X[0]
124 adc \$0, %rdx
125 mov $X[0], $DST
126 ___
127 for(my $i=1;$i<8;$i++) {
128 $code.=<<___;
129 mov %rdx, $TMP
130
131 mov (+8*$i)($SRC2), %rax
132 mul $OP # rdx:rax = %OP * [$i]
133 add %rax, $X[$i]
134 adc \$0, %rdx
135 add $TMP, $X[$i]
136 adc \$0, %rdx
137 ___
138 }
139 $code.=<<___;
140 mov %rdx, $X[0]
141 ___
142 }
143
144 #
145 # Swizzle Macros
146 #
147
148 # macro to copy data from flat space to swizzled table
149 #MACRO swizzle pDst, pSrc, tmp1, tmp2
150 # pDst and pSrc are modified
151 sub swizzle
152 {
153 my ($pDst, $pSrc, $cnt, $d0)=@_;
154 $code.=<<___;
155 mov \$8, $cnt
156 loop_$m:
157 mov ($pSrc), $d0
158 mov $d0#w, ($pDst)
159 shr \$16, $d0
160 mov $d0#w, (+64*1)($pDst)
161 shr \$16, $d0
162 mov $d0#w, (+64*2)($pDst)
163 shr \$16, $d0
164 mov $d0#w, (+64*3)($pDst)
165 lea 8($pSrc), $pSrc
166 lea 64*4($pDst), $pDst
167 dec $cnt
168 jnz loop_$m
169 ___
170
171 $m++;
172 }
173
174 # macro to copy data from swizzled table to flat space
175 #MACRO unswizzle pDst, pSrc, tmp*3
176 sub unswizzle
177 {
178 my ($pDst, $pSrc, $cnt, $d0, $d1)=@_;
179 $code.=<<___;
180 mov \$4, $cnt
181 loop_$m:
182 movzxw (+64*3+256*0)($pSrc), $d0
183 movzxw (+64*3+256*1)($pSrc), $d1
184 shl \$16, $d0
185 shl \$16, $d1
186 mov (+64*2+256*0)($pSrc), $d0#w
187 mov (+64*2+256*1)($pSrc), $d1#w
188 shl \$16, $d0
189 shl \$16, $d1
190 mov (+64*1+256*0)($pSrc), $d0#w
191 mov (+64*1+256*1)($pSrc), $d1#w
192 shl \$16, $d0
193 shl \$16, $d1
194 mov (+64*0+256*0)($pSrc), $d0#w
195 mov (+64*0+256*1)($pSrc), $d1#w
196 mov $d0, (+8*0)($pDst)
197 mov $d1, (+8*1)($pDst)
198 lea 256*2($pSrc), $pSrc
199 lea 8*2($pDst), $pDst
200 sub \$1, $cnt
201 jnz loop_$m
202 ___
203
204 $m++;
205 }
206
207 #
208 # Data Structures
209 #
210
211 # Reduce Data
212 #
213 #
214 # Offset Value
215 # 0C0 Carries
216 # 0B8 X2[10]
217 # 0B0 X2[9]
218 # 0A8 X2[8]
219 # 0A0 X2[7]
220 # 098 X2[6]
221 # 090 X2[5]
222 # 088 X2[4]
223 # 080 X2[3]
224 # 078 X2[2]
225 # 070 X2[1]
226 # 068 X2[0]
227 # 060 X1[12] P[10]
228 # 058 X1[11] P[9] Z[8]
229 # 050 X1[10] P[8] Z[7]
230 # 048 X1[9] P[7] Z[6]
231 # 040 X1[8] P[6] Z[5]
232 # 038 X1[7] P[5] Z[4]
233 # 030 X1[6] P[4] Z[3]
234 # 028 X1[5] P[3] Z[2]
235 # 020 X1[4] P[2] Z[1]
236 # 018 X1[3] P[1] Z[0]
237 # 010 X1[2] P[0] Y[2]
238 # 008 X1[1] Q[1] Y[1]
239 # 000 X1[0] Q[0] Y[0]
240
241 my $X1_offset = 0; # 13 qwords
242 my $X2_offset = $X1_offset + 13*8; # 11 qwords
243 my $Carries_offset = $X2_offset + 11*8; # 1 qword
244 my $Q_offset = 0; # 2 qwords
245 my $P_offset = $Q_offset + 2*8; # 11 qwords
246 my $Y_offset = 0; # 3 qwords
247 my $Z_offset = $Y_offset + 3*8; # 9 qwords
248
249 my $Red_Data_Size = $Carries_offset + 1*8; # (25 qwords)
250
251 #
252 # Stack Frame
253 #
254 #
255 # offset value
256 # ... <old stack contents>
257 # ...
258 # 280 Garray
259
260 # 278 tmp16[15]
261 # ... ...
262 # 200 tmp16[0]
263
264 # 1F8 tmp[7]
265 # ... ...
266 # 1C0 tmp[0]
267
268 # 1B8 GT[7]
269 # ... ...
270 # 180 GT[0]
271
272 # 178 Reduce Data
273 # ... ...
274 # 0B8 Reduce Data
275 # 0B0 reserved
276 # 0A8 reserved
277 # 0A0 reserved
278 # 098 reserved
279 # 090 reserved
280 # 088 reduce result addr
281 # 080 exp[8]
282
283 # ...
284 # 048 exp[1]
285 # 040 exp[0]
286
287 # 038 reserved
288 # 030 loop_idx
289 # 028 pg
290 # 020 i
291 # 018 pData ; arg 4
292 # 010 pG ; arg 2
293 # 008 pResult ; arg 1
294 # 000 rsp ; stack pointer before subtract
295
296 my $rsp_offset = 0;
297 my $pResult_offset = 8*1 + $rsp_offset;
298 my $pG_offset = 8*1 + $pResult_offset;
299 my $pData_offset = 8*1 + $pG_offset;
300 my $i_offset = 8*1 + $pData_offset;
301 my $pg_offset = 8*1 + $i_offset;
302 my $loop_idx_offset = 8*1 + $pg_offset;
303 my $reserved1_offset = 8*1 + $loop_idx_offset;
304 my $exp_offset = 8*1 + $reserved1_offset;
305 my $red_result_addr_offset= 8*9 + $exp_offset;
306 my $reserved2_offset = 8*1 + $red_result_addr_offset;
307 my $Reduce_Data_offset = 8*5 + $reserved2_offset;
308 my $GT_offset = $Red_Data_Size + $Reduce_Data_offset;
309 my $tmp_offset = 8*8 + $GT_offset;
310 my $tmp16_offset = 8*8 + $tmp_offset;
311 my $garray_offset = 8*16 + $tmp16_offset;
312 my $mem_size = 8*8*32 + $garray_offset;
313
314 #
315 # Offsets within Reduce Data
316 #
317 #
318 # struct MODF_2FOLD_MONT_512_C1_DATA {
319 # UINT64 t[8][8];
320 # UINT64 m[8];
321 # UINT64 m1[8]; /* 2^768 % m */
322 # UINT64 m2[8]; /* 2^640 % m */
323 # UINT64 k1[2]; /* (- 1/m) % 2^128 */
324 # };
325
326 my $T = 0;
327 my $M = 512; # = 8 * 8 * 8
328 my $M1 = 576; # = 8 * 8 * 9 /* += 8 * 8 */
329 my $M2 = 640; # = 8 * 8 * 10 /* += 8 * 8 */
330 my $K1 = 704; # = 8 * 8 * 11 /* += 8 * 8 */
331
332 #
333 # FUNCTIONS
334 #
335
336 {{{
337 #
338 # MULADD_128x512 : Function to multiply 128-bits (2 qwords) by 512-bits (8 qwords)
339 # and add 512-bits (8 qwords)
340 # to get 640 bits (10 qwords)
341 # Input: 128-bit mul source: [rdi+8*1], rbp
342 # 512-bit mul source: [rsi+8*n]
343 # 512-bit add source: r15, r14, ..., r9, r8
344 # Output: r9, r8, r15, r14, r13, r12, r11, r10, [rcx+8*1], [rcx+8*0]
345 # Clobbers all regs except: rcx, rsi, rdi
346 $code.=<<___;
347 .type MULADD_128x512,\@abi-omnipotent
348 .align 16
349 MULADD_128x512:
350 ___
351 &MULSTEP_512([map("%r$_",(8..15))], "(+8*0)(%rcx)", "%rsi", "%rbp", "%rbx");
352 $code.=<<___;
353 mov (+8*1)(%rdi), %rbp
354 ___
355 &MULSTEP_512([map("%r$_",(9..15,8))], "(+8*1)(%rcx)", "%rsi", "%rbp", "%rbx");
356 $code.=<<___;
357 ret
358 .size MULADD_128x512,.-MULADD_128x512
359 ___
360 }}}
361
362 {{{
363 #MULADD_256x512 MACRO pDst, pA, pB, OP, TMP, X7, X6, X5, X4, X3, X2, X1, X0
364 #
365 # Inputs: pDst: Destination (768 bits, 12 qwords)
366 # pA: Multiplicand (1024 bits, 16 qwords)
367 # pB: Multiplicand (512 bits, 8 qwords)
368 # Dst = Ah * B + Al
369 # where Ah is (in qwords) A[15:12] (256 bits) and Al is A[7:0] (512 bits)
370 # Results in X3 X2 X1 X0 X7 X6 X5 X4 Dst[3:0]
371 # Uses registers: arguments, RAX, RDX
372 sub MULADD_256x512
373 {
374 my ($pDst, $pA, $pB, $OP, $TMP, $X)=@_;
375 $code.=<<___;
376 mov (+8*12)($pA), $OP
377 ___
378 &MULSTEP_512_ADD($X, "(+8*0)($pDst)", $pB, $pA, $OP, $TMP);
379 push(@$X,shift(@$X));
380
381 $code.=<<___;
382 mov (+8*13)($pA), $OP
383 ___
384 &MULSTEP_512($X, "(+8*1)($pDst)", $pB, $OP, $TMP);
385 push(@$X,shift(@$X));
386
387 $code.=<<___;
388 mov (+8*14)($pA), $OP
389 ___
390 &MULSTEP_512($X, "(+8*2)($pDst)", $pB, $OP, $TMP);
391 push(@$X,shift(@$X));
392
393 $code.=<<___;
394 mov (+8*15)($pA), $OP
395 ___
396 &MULSTEP_512($X, "(+8*3)($pDst)", $pB, $OP, $TMP);
397 push(@$X,shift(@$X));
398 }
399
400 #
401 # mont_reduce(UINT64 *x, /* 1024 bits, 16 qwords */
402 # UINT64 *m, /* 512 bits, 8 qwords */
403 # MODF_2FOLD_MONT_512_C1_DATA *data,
404 # UINT64 *r) /* 512 bits, 8 qwords */
405 # Input: x (number to be reduced): tmp16 (Implicit)
406 # m (modulus): [pM] (Implicit)
407 # data (reduce data): [pData] (Implicit)
408 # Output: r (result): Address in [red_res_addr]
409 # result also in: r9, r8, r15, r14, r13, r12, r11, r10
410
411 my @X=map("%r$_",(8..15));
412
413 $code.=<<___;
414 .type mont_reduce,\@abi-omnipotent
415 .align 16
416 mont_reduce:
417 ___
418
419 my $STACK_DEPTH = 8;
420 #
421 # X1 = Xh * M1 + Xl
422 $code.=<<___;
423 lea (+$Reduce_Data_offset+$X1_offset+$STACK_DEPTH)(%rsp), %rdi # pX1 (Dst) 769 bits, 13 qwords
424 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rsi # pM1 (Bsrc) 512 bits, 8 qwords
425 add \$$M1, %rsi
426 lea (+$tmp16_offset+$STACK_DEPTH)(%rsp), %rcx # X (Asrc) 1024 bits, 16 qwords
427
428 ___
429
430 &MULADD_256x512("%rdi", "%rcx", "%rsi", "%rbp", "%rbx", \@X); # rotates @X 4 times
431 # results in r11, r10, r9, r8, r15, r14, r13, r12, X1[3:0]
432
433 $code.=<<___;
434 xor %rax, %rax
435 # X1 += xl
436 add (+8*8)(%rcx), $X[4]
437 adc (+8*9)(%rcx), $X[5]
438 adc (+8*10)(%rcx), $X[6]
439 adc (+8*11)(%rcx), $X[7]
440 adc \$0, %rax
441 # X1 is now rax, r11-r8, r15-r12, tmp16[3:0]
442
443 #
444 # check for carry ;; carry stored in rax
445 mov $X[4], (+8*8)(%rdi) # rdi points to X1
446 mov $X[5], (+8*9)(%rdi)
447 mov $X[6], %rbp
448 mov $X[7], (+8*11)(%rdi)
449
450 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp)
451
452 mov (+8*0)(%rdi), $X[4]
453 mov (+8*1)(%rdi), $X[5]
454 mov (+8*2)(%rdi), $X[6]
455 mov (+8*3)(%rdi), $X[7]
456
457 # X1 is now stored in: X1[11], rbp, X1[9:8], r15-r8
458 # rdi -> X1
459 # rsi -> M1
460
461 #
462 # X2 = Xh * M2 + Xl
463 # do first part (X2 = Xh * M2)
464 add \$8*10, %rdi # rdi -> pXh ; 128 bits, 2 qwords
465 # Xh is actually { [rdi+8*1], rbp }
466 add \$`$M2-$M1`, %rsi # rsi -> M2
467 lea (+$Reduce_Data_offset+$X2_offset+$STACK_DEPTH)(%rsp), %rcx # rcx -> pX2 ; 641 bits, 11 qwords
468 ___
469 unshift(@X,pop(@X)); unshift(@X,pop(@X));
470 $code.=<<___;
471
472 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8
473 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0]
474 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rax
475
476 # X2 += Xl
477 add (+8*8-8*10)(%rdi), $X[6] # (-8*10) is to adjust rdi -> Xh to Xl
478 adc (+8*9-8*10)(%rdi), $X[7]
479 mov $X[6], (+8*8)(%rcx)
480 mov $X[7], (+8*9)(%rcx)
481
482 adc %rax, %rax
483 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp)
484
485 lea (+$Reduce_Data_offset+$Q_offset+$STACK_DEPTH)(%rsp), %rdi # rdi -> pQ ; 128 bits, 2 qwords
486 add \$`$K1-$M2`, %rsi # rsi -> pK1 ; 128 bits, 2 qwords
487
488 # MUL_128x128t128 rdi, rcx, rsi ; Q = X2 * K1 (bottom half)
489 # B1:B0 = rsi[1:0] = K1[1:0]
490 # A1:A0 = rcx[1:0] = X2[1:0]
491 # Result = rdi[1],rbp = Q[1],rbp
492 mov (%rsi), %r8 # B0
493 mov (+8*1)(%rsi), %rbx # B1
494
495 mov (%rcx), %rax # A0
496 mul %r8 # B0
497 mov %rax, %rbp
498 mov %rdx, %r9
499
500 mov (+8*1)(%rcx), %rax # A1
501 mul %r8 # B0
502 add %rax, %r9
503
504 mov (%rcx), %rax # A0
505 mul %rbx # B1
506 add %rax, %r9
507
508 mov %r9, (+8*1)(%rdi)
509 # end MUL_128x128t128
510
511 sub \$`$K1-$M`, %rsi
512
513 mov (%rcx), $X[6]
514 mov (+8*1)(%rcx), $X[7] # r9:r8 = X2[1:0]
515
516 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8
517 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0]
518
519 # load first half of m to rdx, rdi, rbx, rax
520 # moved this here for efficiency
521 mov (+8*0)(%rsi), %rax
522 mov (+8*1)(%rsi), %rbx
523 mov (+8*2)(%rsi), %rdi
524 mov (+8*3)(%rsi), %rdx
525
526 # continue with reduction
527 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rbp
528
529 add (+8*8)(%rcx), $X[6]
530 adc (+8*9)(%rcx), $X[7]
531
532 #accumulate the final carry to rbp
533 adc %rbp, %rbp
534
535 # Add in overflow corrections: R = (X2>>128) += T[overflow]
536 # R = {r9, r8, r15, r14, ..., r10}
537 shl \$3, %rbp
538 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rcx # rsi -> Data (and points to T)
539 add %rcx, %rbp # pT ; 512 bits, 8 qwords, spread out
540
541 # rsi will be used to generate a mask after the addition
542 xor %rsi, %rsi
543
544 add (+8*8*0)(%rbp), $X[0]
545 adc (+8*8*1)(%rbp), $X[1]
546 adc (+8*8*2)(%rbp), $X[2]
547 adc (+8*8*3)(%rbp), $X[3]
548 adc (+8*8*4)(%rbp), $X[4]
549 adc (+8*8*5)(%rbp), $X[5]
550 adc (+8*8*6)(%rbp), $X[6]
551 adc (+8*8*7)(%rbp), $X[7]
552
553 # if there is a carry: rsi = 0xFFFFFFFFFFFFFFFF
554 # if carry is clear: rsi = 0x0000000000000000
555 sbb \$0, %rsi
556
557 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m
558 and %rsi, %rax
559 and %rsi, %rbx
560 and %rsi, %rdi
561 and %rsi, %rdx
562
563 mov \$1, %rbp
564 sub %rax, $X[0]
565 sbb %rbx, $X[1]
566 sbb %rdi, $X[2]
567 sbb %rdx, $X[3]
568
569 # if there is a borrow: rbp = 0
570 # if there is no borrow: rbp = 1
571 # this is used to save the borrows in between the first half and the 2nd half of the subtraction of m
572 sbb \$0, %rbp
573
574 #load second half of m to rdx, rdi, rbx, rax
575
576 add \$$M, %rcx
577 mov (+8*4)(%rcx), %rax
578 mov (+8*5)(%rcx), %rbx
579 mov (+8*6)(%rcx), %rdi
580 mov (+8*7)(%rcx), %rdx
581
582 # use the rsi mask as before
583 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m
584 and %rsi, %rax
585 and %rsi, %rbx
586 and %rsi, %rdi
587 and %rsi, %rdx
588
589 # if rbp = 0, there was a borrow before, it is moved to the carry flag
590 # if rbp = 1, there was not a borrow before, carry flag is cleared
591 sub \$1, %rbp
592
593 sbb %rax, $X[4]
594 sbb %rbx, $X[5]
595 sbb %rdi, $X[6]
596 sbb %rdx, $X[7]
597
598 # write R back to memory
599
600 mov (+$red_result_addr_offset+$STACK_DEPTH)(%rsp), %rsi
601 mov $X[0], (+8*0)(%rsi)
602 mov $X[1], (+8*1)(%rsi)
603 mov $X[2], (+8*2)(%rsi)
604 mov $X[3], (+8*3)(%rsi)
605 mov $X[4], (+8*4)(%rsi)
606 mov $X[5], (+8*5)(%rsi)
607 mov $X[6], (+8*6)(%rsi)
608 mov $X[7], (+8*7)(%rsi)
609
610 ret
611 .size mont_reduce,.-mont_reduce
612 ___
613 }}}
614
615 {{{
616 #MUL_512x512 MACRO pDst, pA, pB, x7, x6, x5, x4, x3, x2, x1, x0, tmp*2
617 #
618 # Inputs: pDst: Destination (1024 bits, 16 qwords)
619 # pA: Multiplicand (512 bits, 8 qwords)
620 # pB: Multiplicand (512 bits, 8 qwords)
621 # Uses registers rax, rdx, args
622 # B operand in [pB] and also in x7...x0
623 sub MUL_512x512
624 {
625 my ($pDst, $pA, $pB, $x, $OP, $TMP, $pDst_o)=@_;
626 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/);
627 my @X=@$x; # make a copy
628
629 $code.=<<___;
630 mov (+8*0)($pA), $OP
631
632 mov $X[0], %rax
633 mul $OP # rdx:rax = %OP * [0]
634 mov %rax, (+$pDst_o+8*0)($pDst)
635 mov %rdx, $X[0]
636 ___
637 for(my $i=1;$i<8;$i++) {
638 $code.=<<___;
639 mov $X[$i], %rax
640 mul $OP # rdx:rax = %OP * [$i]
641 add %rax, $X[$i-1]
642 adc \$0, %rdx
643 mov %rdx, $X[$i]
644 ___
645 }
646
647 for(my $i=1;$i<8;$i++) {
648 $code.=<<___;
649 mov (+8*$i)($pA), $OP
650 ___
651
652 &MULSTEP_512(\@X, "(+$pDst_o+8*$i)($pDst)", $pB, $OP, $TMP);
653 push(@X,shift(@X));
654 }
655
656 $code.=<<___;
657 mov $X[0], (+$pDst_o+8*8)($pDst)
658 mov $X[1], (+$pDst_o+8*9)($pDst)
659 mov $X[2], (+$pDst_o+8*10)($pDst)
660 mov $X[3], (+$pDst_o+8*11)($pDst)
661 mov $X[4], (+$pDst_o+8*12)($pDst)
662 mov $X[5], (+$pDst_o+8*13)($pDst)
663 mov $X[6], (+$pDst_o+8*14)($pDst)
664 mov $X[7], (+$pDst_o+8*15)($pDst)
665 ___
666 }
667
668 #
669 # mont_mul_a3b : subroutine to compute (Src1 * Src2) % M (all 512-bits)
670 # Input: src1: Address of source 1: rdi
671 # src2: Address of source 2: rsi
672 # Output: dst: Address of destination: [red_res_addr]
673 # src2 and result also in: r9, r8, r15, r14, r13, r12, r11, r10
674 # Temp: Clobbers [tmp16], all registers
675 $code.=<<___;
676 .type mont_mul_a3b,\@abi-omnipotent
677 .align 16
678 mont_mul_a3b:
679 #
680 # multiply tmp = src1 * src2
681 # For multiply: dst = rcx, src1 = rdi, src2 = rsi
682 # stack depth is extra 8 from call
683 ___
684 &MUL_512x512("%rsp+$tmp16_offset+8", "%rdi", "%rsi", [map("%r$_",(10..15,8..9))], "%rbp", "%rbx");
685 $code.=<<___;
686 #
687 # Dst = tmp % m
688 # Call reduce(tmp, m, data, dst)
689
690 # tail recursion optimization: jmp to mont_reduce and return from there
691 jmp mont_reduce
692 # call mont_reduce
693 # ret
694 .size mont_mul_a3b,.-mont_mul_a3b
695 ___
696 }}}
697
698 {{{
699 #SQR_512 MACRO pDest, pA, x7, x6, x5, x4, x3, x2, x1, x0, tmp*4
700 #
701 # Input in memory [pA] and also in x7...x0
702 # Uses all argument registers plus rax and rdx
703 #
704 # This version computes all of the off-diagonal terms into memory,
705 # and then it adds in the diagonal terms
706
707 sub SQR_512
708 {
709 my ($pDst, $pA, $x, $A, $tmp, $x7, $x6, $pDst_o)=@_;
710 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/);
711 my @X=@$x; # make a copy
712 $code.=<<___;
713 # ------------------
714 # first pass 01...07
715 # ------------------
716 mov $X[0], $A
717
718 mov $X[1],%rax
719 mul $A
720 mov %rax, (+$pDst_o+8*1)($pDst)
721 ___
722 for(my $i=2;$i<8;$i++) {
723 $code.=<<___;
724 mov %rdx, $X[$i-2]
725 mov $X[$i],%rax
726 mul $A
727 add %rax, $X[$i-2]
728 adc \$0, %rdx
729 ___
730 }
731 $code.=<<___;
732 mov %rdx, $x7
733
734 mov $X[0], (+$pDst_o+8*2)($pDst)
735
736 # ------------------
737 # second pass 12...17
738 # ------------------
739
740 mov (+8*1)($pA), $A
741
742 mov (+8*2)($pA),%rax
743 mul $A
744 add %rax, $X[1]
745 adc \$0, %rdx
746 mov $X[1], (+$pDst_o+8*3)($pDst)
747
748 mov %rdx, $X[0]
749 mov (+8*3)($pA),%rax
750 mul $A
751 add %rax, $X[2]
752 adc \$0, %rdx
753 add $X[0], $X[2]
754 adc \$0, %rdx
755 mov $X[2], (+$pDst_o+8*4)($pDst)
756
757 mov %rdx, $X[0]
758 mov (+8*4)($pA),%rax
759 mul $A
760 add %rax, $X[3]
761 adc \$0, %rdx
762 add $X[0], $X[3]
763 adc \$0, %rdx
764
765 mov %rdx, $X[0]
766 mov (+8*5)($pA),%rax
767 mul $A
768 add %rax, $X[4]
769 adc \$0, %rdx
770 add $X[0], $X[4]
771 adc \$0, %rdx
772
773 mov %rdx, $X[0]
774 mov $X[6],%rax
775 mul $A
776 add %rax, $X[5]
777 adc \$0, %rdx
778 add $X[0], $X[5]
779 adc \$0, %rdx
780
781 mov %rdx, $X[0]
782 mov $X[7],%rax
783 mul $A
784 add %rax, $x7
785 adc \$0, %rdx
786 add $X[0], $x7
787 adc \$0, %rdx
788
789 mov %rdx, $X[1]
790
791 # ------------------
792 # third pass 23...27
793 # ------------------
794 mov (+8*2)($pA), $A
795
796 mov (+8*3)($pA),%rax
797 mul $A
798 add %rax, $X[3]
799 adc \$0, %rdx
800 mov $X[3], (+$pDst_o+8*5)($pDst)
801
802 mov %rdx, $X[0]
803 mov (+8*4)($pA),%rax
804 mul $A
805 add %rax, $X[4]
806 adc \$0, %rdx
807 add $X[0], $X[4]
808 adc \$0, %rdx
809 mov $X[4], (+$pDst_o+8*6)($pDst)
810
811 mov %rdx, $X[0]
812 mov (+8*5)($pA),%rax
813 mul $A
814 add %rax, $X[5]
815 adc \$0, %rdx
816 add $X[0], $X[5]
817 adc \$0, %rdx
818
819 mov %rdx, $X[0]
820 mov $X[6],%rax
821 mul $A
822 add %rax, $x7
823 adc \$0, %rdx
824 add $X[0], $x7
825 adc \$0, %rdx
826
827 mov %rdx, $X[0]
828 mov $X[7],%rax
829 mul $A
830 add %rax, $X[1]
831 adc \$0, %rdx
832 add $X[0], $X[1]
833 adc \$0, %rdx
834
835 mov %rdx, $X[2]
836
837 # ------------------
838 # fourth pass 34...37
839 # ------------------
840
841 mov (+8*3)($pA), $A
842
843 mov (+8*4)($pA),%rax
844 mul $A
845 add %rax, $X[5]
846 adc \$0, %rdx
847 mov $X[5], (+$pDst_o+8*7)($pDst)
848
849 mov %rdx, $X[0]
850 mov (+8*5)($pA),%rax
851 mul $A
852 add %rax, $x7
853 adc \$0, %rdx
854 add $X[0], $x7
855 adc \$0, %rdx
856 mov $x7, (+$pDst_o+8*8)($pDst)
857
858 mov %rdx, $X[0]
859 mov $X[6],%rax
860 mul $A
861 add %rax, $X[1]
862 adc \$0, %rdx
863 add $X[0], $X[1]
864 adc \$0, %rdx
865
866 mov %rdx, $X[0]
867 mov $X[7],%rax
868 mul $A
869 add %rax, $X[2]
870 adc \$0, %rdx
871 add $X[0], $X[2]
872 adc \$0, %rdx
873
874 mov %rdx, $X[5]
875
876 # ------------------
877 # fifth pass 45...47
878 # ------------------
879 mov (+8*4)($pA), $A
880
881 mov (+8*5)($pA),%rax
882 mul $A
883 add %rax, $X[1]
884 adc \$0, %rdx
885 mov $X[1], (+$pDst_o+8*9)($pDst)
886
887 mov %rdx, $X[0]
888 mov $X[6],%rax
889 mul $A
890 add %rax, $X[2]
891 adc \$0, %rdx
892 add $X[0], $X[2]
893 adc \$0, %rdx
894 mov $X[2], (+$pDst_o+8*10)($pDst)
895
896 mov %rdx, $X[0]
897 mov $X[7],%rax
898 mul $A
899 add %rax, $X[5]
900 adc \$0, %rdx
901 add $X[0], $X[5]
902 adc \$0, %rdx
903
904 mov %rdx, $X[1]
905
906 # ------------------
907 # sixth pass 56...57
908 # ------------------
909 mov (+8*5)($pA), $A
910
911 mov $X[6],%rax
912 mul $A
913 add %rax, $X[5]
914 adc \$0, %rdx
915 mov $X[5], (+$pDst_o+8*11)($pDst)
916
917 mov %rdx, $X[0]
918 mov $X[7],%rax
919 mul $A
920 add %rax, $X[1]
921 adc \$0, %rdx
922 add $X[0], $X[1]
923 adc \$0, %rdx
924 mov $X[1], (+$pDst_o+8*12)($pDst)
925
926 mov %rdx, $X[2]
927
928 # ------------------
929 # seventh pass 67
930 # ------------------
931 mov $X[6], $A
932
933 mov $X[7],%rax
934 mul $A
935 add %rax, $X[2]
936 adc \$0, %rdx
937 mov $X[2], (+$pDst_o+8*13)($pDst)
938
939 mov %rdx, (+$pDst_o+8*14)($pDst)
940
941 # start finalize (add in squares, and double off-terms)
942 mov (+$pDst_o+8*1)($pDst), $X[0]
943 mov (+$pDst_o+8*2)($pDst), $X[1]
944 mov (+$pDst_o+8*3)($pDst), $X[2]
945 mov (+$pDst_o+8*4)($pDst), $X[3]
946 mov (+$pDst_o+8*5)($pDst), $X[4]
947 mov (+$pDst_o+8*6)($pDst), $X[5]
948
949 mov (+8*3)($pA), %rax
950 mul %rax
951 mov %rax, $x6
952 mov %rdx, $X[6]
953
954 add $X[0], $X[0]
955 adc $X[1], $X[1]
956 adc $X[2], $X[2]
957 adc $X[3], $X[3]
958 adc $X[4], $X[4]
959 adc $X[5], $X[5]
960 adc \$0, $X[6]
961
962 mov (+8*0)($pA), %rax
963 mul %rax
964 mov %rax, (+$pDst_o+8*0)($pDst)
965 mov %rdx, $A
966
967 mov (+8*1)($pA), %rax
968 mul %rax
969
970 add $A, $X[0]
971 adc %rax, $X[1]
972 adc \$0, %rdx
973
974 mov %rdx, $A
975 mov $X[0], (+$pDst_o+8*1)($pDst)
976 mov $X[1], (+$pDst_o+8*2)($pDst)
977
978 mov (+8*2)($pA), %rax
979 mul %rax
980
981 add $A, $X[2]
982 adc %rax, $X[3]
983 adc \$0, %rdx
984
985 mov %rdx, $A
986
987 mov $X[2], (+$pDst_o+8*3)($pDst)
988 mov $X[3], (+$pDst_o+8*4)($pDst)
989
990 xor $tmp, $tmp
991 add $A, $X[4]
992 adc $x6, $X[5]
993 adc \$0, $tmp
994
995 mov $X[4], (+$pDst_o+8*5)($pDst)
996 mov $X[5], (+$pDst_o+8*6)($pDst)
997
998 # %%tmp has 0/1 in column 7
999 # %%A6 has a full value in column 7
1000
1001 mov (+$pDst_o+8*7)($pDst), $X[0]
1002 mov (+$pDst_o+8*8)($pDst), $X[1]
1003 mov (+$pDst_o+8*9)($pDst), $X[2]
1004 mov (+$pDst_o+8*10)($pDst), $X[3]
1005 mov (+$pDst_o+8*11)($pDst), $X[4]
1006 mov (+$pDst_o+8*12)($pDst), $X[5]
1007 mov (+$pDst_o+8*13)($pDst), $x6
1008 mov (+$pDst_o+8*14)($pDst), $x7
1009
1010 mov $X[7], %rax
1011 mul %rax
1012 mov %rax, $X[7]
1013 mov %rdx, $A
1014
1015 add $X[0], $X[0]
1016 adc $X[1], $X[1]
1017 adc $X[2], $X[2]
1018 adc $X[3], $X[3]
1019 adc $X[4], $X[4]
1020 adc $X[5], $X[5]
1021 adc $x6, $x6
1022 adc $x7, $x7
1023 adc \$0, $A
1024
1025 add $tmp, $X[0]
1026
1027 mov (+8*4)($pA), %rax
1028 mul %rax
1029
1030 add $X[6], $X[0]
1031 adc %rax, $X[1]
1032 adc \$0, %rdx
1033
1034 mov %rdx, $tmp
1035
1036 mov $X[0], (+$pDst_o+8*7)($pDst)
1037 mov $X[1], (+$pDst_o+8*8)($pDst)
1038
1039 mov (+8*5)($pA), %rax
1040 mul %rax
1041
1042 add $tmp, $X[2]
1043 adc %rax, $X[3]
1044 adc \$0, %rdx
1045
1046 mov %rdx, $tmp
1047
1048 mov $X[2], (+$pDst_o+8*9)($pDst)
1049 mov $X[3], (+$pDst_o+8*10)($pDst)
1050
1051 mov (+8*6)($pA), %rax
1052 mul %rax
1053
1054 add $tmp, $X[4]
1055 adc %rax, $X[5]
1056 adc \$0, %rdx
1057
1058 mov $X[4], (+$pDst_o+8*11)($pDst)
1059 mov $X[5], (+$pDst_o+8*12)($pDst)
1060
1061 add %rdx, $x6
1062 adc $X[7], $x7
1063 adc \$0, $A
1064
1065 mov $x6, (+$pDst_o+8*13)($pDst)
1066 mov $x7, (+$pDst_o+8*14)($pDst)
1067 mov $A, (+$pDst_o+8*15)($pDst)
1068 ___
1069 }
1070
1071 #
1072 # sqr_reduce: subroutine to compute Result = reduce(Result * Result)
1073 #
1074 # input and result also in: r9, r8, r15, r14, r13, r12, r11, r10
1075 #
1076 $code.=<<___;
1077 .type sqr_reduce,\@abi-omnipotent
1078 .align 16
1079 sqr_reduce:
1080 mov (+$pResult_offset+8)(%rsp), %rcx
1081 ___
1082 &SQR_512("%rsp+$tmp16_offset+8", "%rcx", [map("%r$_",(10..15,8..9))], "%rbx", "%rbp", "%rsi", "%rdi");
1083 $code.=<<___;
1084 # tail recursion optimization: jmp to mont_reduce and return from there
1085 jmp mont_reduce
1086 # call mont_reduce
1087 # ret
1088 .size sqr_reduce,.-sqr_reduce
1089 ___
1090 }}}
1091
1092 #
1093 # MAIN FUNCTION
1094 #
1095
1096 #mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */
1097 # UINT64 *g, /* 512 bits, 8 qwords */
1098 # UINT64 *exp, /* 512 bits, 8 qwords */
1099 # struct mod_ctx_512 *data)
1100
1101 # window size = 5
1102 # table size = 2^5 = 32
1103 #table_entries equ 32
1104 #table_size equ table_entries * 8
1105 $code.=<<___;
1106 .globl mod_exp_512
1107 .type mod_exp_512,\@function,4
1108 mod_exp_512:
1109 push %rbp
1110 push %rbx
1111 push %r12
1112 push %r13
1113 push %r14
1114 push %r15
1115
1116 # adjust stack down and then align it with cache boundary
1117 mov %rsp, %r8
1118 sub \$$mem_size, %rsp
1119 and \$-64, %rsp
1120
1121 # store previous stack pointer and arguments
1122 mov %r8, (+$rsp_offset)(%rsp)
1123 mov %rdi, (+$pResult_offset)(%rsp)
1124 mov %rsi, (+$pG_offset)(%rsp)
1125 mov %rcx, (+$pData_offset)(%rsp)
1126 .Lbody:
1127 # transform g into montgomery space
1128 # GT = reduce(g * C2) = reduce(g * (2^256))
1129 # reduce expects to have the input in [tmp16]
1130 pxor %xmm4, %xmm4
1131 movdqu (+16*0)(%rsi), %xmm0
1132 movdqu (+16*1)(%rsi), %xmm1
1133 movdqu (+16*2)(%rsi), %xmm2
1134 movdqu (+16*3)(%rsi), %xmm3
1135 movdqa %xmm4, (+$tmp16_offset+16*0)(%rsp)
1136 movdqa %xmm4, (+$tmp16_offset+16*1)(%rsp)
1137 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp)
1138 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp)
1139 movdqa %xmm0, (+$tmp16_offset+16*2)(%rsp)
1140 movdqa %xmm1, (+$tmp16_offset+16*3)(%rsp)
1141 movdqa %xmm2, (+$tmp16_offset+16*4)(%rsp)
1142 movdqa %xmm3, (+$tmp16_offset+16*5)(%rsp)
1143
1144 # load pExp before rdx gets blown away
1145 movdqu (+16*0)(%rdx), %xmm0
1146 movdqu (+16*1)(%rdx), %xmm1
1147 movdqu (+16*2)(%rdx), %xmm2
1148 movdqu (+16*3)(%rdx), %xmm3
1149
1150 lea (+$GT_offset)(%rsp), %rbx
1151 mov %rbx, (+$red_result_addr_offset)(%rsp)
1152 call mont_reduce
1153
1154 # Initialize tmp = C
1155 lea (+$tmp_offset)(%rsp), %rcx
1156 xor %rax, %rax
1157 mov %rax, (+8*0)(%rcx)
1158 mov %rax, (+8*1)(%rcx)
1159 mov %rax, (+8*3)(%rcx)
1160 mov %rax, (+8*4)(%rcx)
1161 mov %rax, (+8*5)(%rcx)
1162 mov %rax, (+8*6)(%rcx)
1163 mov %rax, (+8*7)(%rcx)
1164 mov %rax, (+$exp_offset+8*8)(%rsp)
1165 movq \$1, (+8*2)(%rcx)
1166
1167 lea (+$garray_offset)(%rsp), %rbp
1168 mov %rcx, %rsi # pTmp
1169 mov %rbp, %rdi # Garray[][0]
1170 ___
1171
1172 &swizzle("%rdi", "%rcx", "%rax", "%rbx");
1173
1174 # for (rax = 31; rax != 0; rax--) {
1175 # tmp = reduce(tmp * G)
1176 # swizzle(pg, tmp);
1177 # pg += 2; }
1178 $code.=<<___;
1179 mov \$31, %rax
1180 mov %rax, (+$i_offset)(%rsp)
1181 mov %rbp, (+$pg_offset)(%rsp)
1182 # rsi -> pTmp
1183 mov %rsi, (+$red_result_addr_offset)(%rsp)
1184 mov (+8*0)(%rsi), %r10
1185 mov (+8*1)(%rsi), %r11
1186 mov (+8*2)(%rsi), %r12
1187 mov (+8*3)(%rsi), %r13
1188 mov (+8*4)(%rsi), %r14
1189 mov (+8*5)(%rsi), %r15
1190 mov (+8*6)(%rsi), %r8
1191 mov (+8*7)(%rsi), %r9
1192 init_loop:
1193 lea (+$GT_offset)(%rsp), %rdi
1194 call mont_mul_a3b
1195 lea (+$tmp_offset)(%rsp), %rsi
1196 mov (+$pg_offset)(%rsp), %rbp
1197 add \$2, %rbp
1198 mov %rbp, (+$pg_offset)(%rsp)
1199 mov %rsi, %rcx # rcx = rsi = addr of tmp
1200 ___
1201
1202 &swizzle("%rbp", "%rcx", "%rax", "%rbx");
1203 $code.=<<___;
1204 mov (+$i_offset)(%rsp), %rax
1205 sub \$1, %rax
1206 mov %rax, (+$i_offset)(%rsp)
1207 jne init_loop
1208
1209 #
1210 # Copy exponent onto stack
1211 movdqa %xmm0, (+$exp_offset+16*0)(%rsp)
1212 movdqa %xmm1, (+$exp_offset+16*1)(%rsp)
1213 movdqa %xmm2, (+$exp_offset+16*2)(%rsp)
1214 movdqa %xmm3, (+$exp_offset+16*3)(%rsp)
1215
1216
1217 #
1218 # Do exponentiation
1219 # Initialize result to G[exp{511:507}]
1220 mov (+$exp_offset+62)(%rsp), %eax
1221 mov %rax, %rdx
1222 shr \$11, %rax
1223 and \$0x07FF, %edx
1224 mov %edx, (+$exp_offset+62)(%rsp)
1225 lea (+$garray_offset)(%rsp,%rax,2), %rsi
1226 mov (+$pResult_offset)(%rsp), %rdx
1227 ___
1228
1229 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax");
1230
1231 #
1232 # Loop variables
1233 # rcx = [loop_idx] = index: 510-5 to 0 by 5
1234 $code.=<<___;
1235 movq \$505, (+$loop_idx_offset)(%rsp)
1236
1237 mov (+$pResult_offset)(%rsp), %rcx
1238 mov %rcx, (+$red_result_addr_offset)(%rsp)
1239 mov (+8*0)(%rcx), %r10
1240 mov (+8*1)(%rcx), %r11
1241 mov (+8*2)(%rcx), %r12
1242 mov (+8*3)(%rcx), %r13
1243 mov (+8*4)(%rcx), %r14
1244 mov (+8*5)(%rcx), %r15
1245 mov (+8*6)(%rcx), %r8
1246 mov (+8*7)(%rcx), %r9
1247 jmp sqr_2
1248
1249 main_loop_a3b:
1250 call sqr_reduce
1251 call sqr_reduce
1252 call sqr_reduce
1253 sqr_2:
1254 call sqr_reduce
1255 call sqr_reduce
1256
1257 #
1258 # Do multiply, first look up proper value in Garray
1259 mov (+$loop_idx_offset)(%rsp), %rcx # bit index
1260 mov %rcx, %rax
1261 shr \$4, %rax # rax is word pointer
1262 mov (+$exp_offset)(%rsp,%rax,2), %edx
1263 and \$15, %rcx
1264 shrq %cl, %rdx
1265 and \$0x1F, %rdx
1266
1267 lea (+$garray_offset)(%rsp,%rdx,2), %rsi
1268 lea (+$tmp_offset)(%rsp), %rdx
1269 mov %rdx, %rdi
1270 ___
1271
1272 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax");
1273 # rdi = tmp = pG
1274
1275 #
1276 # Call mod_mul_a1(pDst, pSrc1, pSrc2, pM, pData)
1277 # result result pG M Data
1278 $code.=<<___;
1279 mov (+$pResult_offset)(%rsp), %rsi
1280 call mont_mul_a3b
1281
1282 #
1283 # finish loop
1284 mov (+$loop_idx_offset)(%rsp), %rcx
1285 sub \$5, %rcx
1286 mov %rcx, (+$loop_idx_offset)(%rsp)
1287 jge main_loop_a3b
1288
1289 #
1290
1291 end_main_loop_a3b:
1292 # transform result out of Montgomery space
1293 # result = reduce(result)
1294 mov (+$pResult_offset)(%rsp), %rdx
1295 pxor %xmm4, %xmm4
1296 movdqu (+16*0)(%rdx), %xmm0
1297 movdqu (+16*1)(%rdx), %xmm1
1298 movdqu (+16*2)(%rdx), %xmm2
1299 movdqu (+16*3)(%rdx), %xmm3
1300 movdqa %xmm4, (+$tmp16_offset+16*4)(%rsp)
1301 movdqa %xmm4, (+$tmp16_offset+16*5)(%rsp)
1302 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp)
1303 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp)
1304 movdqa %xmm0, (+$tmp16_offset+16*0)(%rsp)
1305 movdqa %xmm1, (+$tmp16_offset+16*1)(%rsp)
1306 movdqa %xmm2, (+$tmp16_offset+16*2)(%rsp)
1307 movdqa %xmm3, (+$tmp16_offset+16*3)(%rsp)
1308 call mont_reduce
1309
1310 # If result > m, subract m
1311 # load result into r15:r8
1312 mov (+$pResult_offset)(%rsp), %rax
1313 mov (+8*0)(%rax), %r8
1314 mov (+8*1)(%rax), %r9
1315 mov (+8*2)(%rax), %r10
1316 mov (+8*3)(%rax), %r11
1317 mov (+8*4)(%rax), %r12
1318 mov (+8*5)(%rax), %r13
1319 mov (+8*6)(%rax), %r14
1320 mov (+8*7)(%rax), %r15
1321
1322 # subtract m
1323 mov (+$pData_offset)(%rsp), %rbx
1324 add \$$M, %rbx
1325
1326 sub (+8*0)(%rbx), %r8
1327 sbb (+8*1)(%rbx), %r9
1328 sbb (+8*2)(%rbx), %r10
1329 sbb (+8*3)(%rbx), %r11
1330 sbb (+8*4)(%rbx), %r12
1331 sbb (+8*5)(%rbx), %r13
1332 sbb (+8*6)(%rbx), %r14
1333 sbb (+8*7)(%rbx), %r15
1334
1335 # if Carry is clear, replace result with difference
1336 mov (+8*0)(%rax), %rsi
1337 mov (+8*1)(%rax), %rdi
1338 mov (+8*2)(%rax), %rcx
1339 mov (+8*3)(%rax), %rdx
1340 cmovnc %r8, %rsi
1341 cmovnc %r9, %rdi
1342 cmovnc %r10, %rcx
1343 cmovnc %r11, %rdx
1344 mov %rsi, (+8*0)(%rax)
1345 mov %rdi, (+8*1)(%rax)
1346 mov %rcx, (+8*2)(%rax)
1347 mov %rdx, (+8*3)(%rax)
1348
1349 mov (+8*4)(%rax), %rsi
1350 mov (+8*5)(%rax), %rdi
1351 mov (+8*6)(%rax), %rcx
1352 mov (+8*7)(%rax), %rdx
1353 cmovnc %r12, %rsi
1354 cmovnc %r13, %rdi
1355 cmovnc %r14, %rcx
1356 cmovnc %r15, %rdx
1357 mov %rsi, (+8*4)(%rax)
1358 mov %rdi, (+8*5)(%rax)
1359 mov %rcx, (+8*6)(%rax)
1360 mov %rdx, (+8*7)(%rax)
1361
1362 mov (+$rsp_offset)(%rsp), %rsi
1363 mov 0(%rsi),%r15
1364 mov 8(%rsi),%r14
1365 mov 16(%rsi),%r13
1366 mov 24(%rsi),%r12
1367 mov 32(%rsi),%rbx
1368 mov 40(%rsi),%rbp
1369 lea 48(%rsi),%rsp
1370 .Lepilogue:
1371 ret
1372 .size mod_exp_512, . - mod_exp_512
1373 ___
1374
1375 sub reg_part {
1376 my ($reg,$conv)=@_;
1377 if ($reg =~ /%r[0-9]+/) { $reg .= $conv; }
1378 elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; }
1379 elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; }
1380 elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; }
1381 return $reg;
1382 }
1383
1384 $code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem;
1385 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1386 $code =~ s/(\(\+[^)]+\))/eval $1/gem;
1387 print $code;
1388 close STDOUT;
Unleashed OS