1 #!/usr/bin/env perl
2 #
3 # Copyright (c) 2010-2011 Intel Corp.
4 # Author: Vinodh.Gopal@intel.com
5 # Jim Guilford
6 # Erdinc.Ozturk@intel.com
7 # Maxim.Perminov@intel.com
8 #
9 # More information about algorithm used can be found at:
10 # http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf
11 #
12 # ====================================================================
13 # Copyright (c) 2011 The OpenSSL Project. All rights reserved.
14 #
15 # Redistribution and use in source and binary forms, with or without
16 # modification, are permitted provided that the following conditions
17 # are met:
18 #
19 # 1. Redistributions of source code must retain the above copyright
20 # notice, this list of conditions and the following disclaimer.
21 #
22 # 2. Redistributions in binary form must reproduce the above copyright
23 # notice, this list of conditions and the following disclaimer in
24 # the documentation and/or other materials provided with the
25 # distribution.
26 #
27 # 3. All advertising materials mentioning features or use of this
28 # software must display the following acknowledgment:
29 # "This product includes software developed by the OpenSSL Project
30 # for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
31 #
32 # 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
33 # endorse or promote products derived from this software without
34 # prior written permission. For written permission, please contact
35 # licensing@OpenSSL.org.
36 #
37 # 5. Products derived from this software may not be called "OpenSSL"
38 # nor may "OpenSSL" appear in their names without prior written
39 # permission of the OpenSSL Project.
40 #
41 # 6. Redistributions of any form whatsoever must retain the following
42 # acknowledgment:
43 # "This product includes software developed by the OpenSSL Project
44 # for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
45 #
46 # THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
47 # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
49 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
50 # ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
51 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
52 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
53 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
54 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
55 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
56 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
57 # OF THE POSSIBILITY OF SUCH DAMAGE.
58 # ====================================================================
59
60 $flavour = shift;
61 $output = shift;
62 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
63
64 my $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
65
66 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
67 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
68 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
69 die "can't locate x86_64-xlate.pl";
70
71 open OUT,"| \"$^X\" $xlate $flavour $output";
72 *STDOUT=*OUT;
73
74 use strict;
75 my $code=".text\n\n";
76 my $m=0;
77
78 #
79 # Define x512 macros
80 #
81
82 #MULSTEP_512_ADD MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src1, src2, add_src, tmp1, tmp2
83 #
84 # uses rax, rdx, and args
85 sub MULSTEP_512_ADD
86 {
87 my ($x, $DST, $SRC2, $ASRC, $OP, $TMP)=@_;
88 my @X=@$x; # make a copy
89 $code.=<<___;
90 mov (+8*0)($SRC2), %rax
91 mul $OP # rdx:rax = %OP * [0]
92 mov ($ASRC), $X[0]
93 add %rax, $X[0]
94 adc \$0, %rdx
95 mov $X[0], $DST
96 ___
97 for(my $i=1;$i<8;$i++) {
98 $code.=<<___;
99 mov %rdx, $TMP
100
101 mov (+8*$i)($SRC2), %rax
102 mul $OP # rdx:rax = %OP * [$i]
103 mov (+8*$i)($ASRC), $X[$i]
104 add %rax, $X[$i]
105 adc \$0, %rdx
106 add $TMP, $X[$i]
107 adc \$0, %rdx
108 ___
109 }
110 $code.=<<___;
111 mov %rdx, $X[0]
112 ___
113 }
114
115 #MULSTEP_512 MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src2, src1_val, tmp
116 #
117 # uses rax, rdx, and args
118 sub MULSTEP_512
119 {
120 my ($x, $DST, $SRC2, $OP, $TMP)=@_;
121 my @X=@$x; # make a copy
122 $code.=<<___;
123 mov (+8*0)($SRC2), %rax
124 mul $OP # rdx:rax = %OP * [0]
125 add %rax, $X[0]
126 adc \$0, %rdx
127 mov $X[0], $DST
128 ___
129 for(my $i=1;$i<8;$i++) {
130 $code.=<<___;
131 mov %rdx, $TMP
132
133 mov (+8*$i)($SRC2), %rax
134 mul $OP # rdx:rax = %OP * [$i]
135 add %rax, $X[$i]
136 adc \$0, %rdx
137 add $TMP, $X[$i]
138 adc \$0, %rdx
139 ___
140 }
141 $code.=<<___;
142 mov %rdx, $X[0]
143 ___
144 }
145
146 #
147 # Swizzle Macros
148 #
149
150 # macro to copy data from flat space to swizzled table
151 #MACRO swizzle pDst, pSrc, tmp1, tmp2
152 # pDst and pSrc are modified
153 sub swizzle
154 {
155 my ($pDst, $pSrc, $cnt, $d0)=@_;
156 $code.=<<___;
157 mov \$8, $cnt
158 loop_$m:
159 mov ($pSrc), $d0
160 mov $d0#w, ($pDst)
161 shr \$16, $d0
162 mov $d0#w, (+64*1)($pDst)
163 shr \$16, $d0
164 mov $d0#w, (+64*2)($pDst)
165 shr \$16, $d0
166 mov $d0#w, (+64*3)($pDst)
167 lea 8($pSrc), $pSrc
168 lea 64*4($pDst), $pDst
169 dec $cnt
170 jnz loop_$m
171 ___
172
173 $m++;
174 }
175
176 # macro to copy data from swizzled table to flat space
177 #MACRO unswizzle pDst, pSrc, tmp*3
178 sub unswizzle
179 {
180 my ($pDst, $pSrc, $cnt, $d0, $d1)=@_;
181 $code.=<<___;
182 mov \$4, $cnt
183 loop_$m:
184 movzxw (+64*3+256*0)($pSrc), $d0
185 movzxw (+64*3+256*1)($pSrc), $d1
186 shl \$16, $d0
187 shl \$16, $d1
188 mov (+64*2+256*0)($pSrc), $d0#w
189 mov (+64*2+256*1)($pSrc), $d1#w
190 shl \$16, $d0
191 shl \$16, $d1
192 mov (+64*1+256*0)($pSrc), $d0#w
193 mov (+64*1+256*1)($pSrc), $d1#w
194 shl \$16, $d0
195 shl \$16, $d1
196 mov (+64*0+256*0)($pSrc), $d0#w
197 mov (+64*0+256*1)($pSrc), $d1#w
198 mov $d0, (+8*0)($pDst)
199 mov $d1, (+8*1)($pDst)
200 lea 256*2($pSrc), $pSrc
201 lea 8*2($pDst), $pDst
202 sub \$1, $cnt
203 jnz loop_$m
204 ___
205
206 $m++;
207 }
208
209 #
210 # Data Structures
211 #
212
213 # Reduce Data
214 #
215 #
216 # Offset Value
217 # 0C0 Carries
218 # 0B8 X2[10]
219 # 0B0 X2[9]
220 # 0A8 X2[8]
221 # 0A0 X2[7]
222 # 098 X2[6]
223 # 090 X2[5]
224 # 088 X2[4]
225 # 080 X2[3]
226 # 078 X2[2]
227 # 070 X2[1]
228 # 068 X2[0]
229 # 060 X1[12] P[10]
230 # 058 X1[11] P[9] Z[8]
231 # 050 X1[10] P[8] Z[7]
232 # 048 X1[9] P[7] Z[6]
233 # 040 X1[8] P[6] Z[5]
234 # 038 X1[7] P[5] Z[4]
235 # 030 X1[6] P[4] Z[3]
236 # 028 X1[5] P[3] Z[2]
237 # 020 X1[4] P[2] Z[1]
238 # 018 X1[3] P[1] Z[0]
239 # 010 X1[2] P[0] Y[2]
240 # 008 X1[1] Q[1] Y[1]
241 # 000 X1[0] Q[0] Y[0]
242
243 my $X1_offset = 0; # 13 qwords
244 my $X2_offset = $X1_offset + 13*8; # 11 qwords
245 my $Carries_offset = $X2_offset + 11*8; # 1 qword
246 my $Q_offset = 0; # 2 qwords
247 my $P_offset = $Q_offset + 2*8; # 11 qwords
248 my $Y_offset = 0; # 3 qwords
249 my $Z_offset = $Y_offset + 3*8; # 9 qwords
250
251 my $Red_Data_Size = $Carries_offset + 1*8; # (25 qwords)
252
253 #
254 # Stack Frame
255 #
256 #
257 # offset value
258 # ... <old stack contents>
259 # ...
260 # 280 Garray
261
262 # 278 tmp16[15]
263 # ... ...
264 # 200 tmp16[0]
265
266 # 1F8 tmp[7]
267 # ... ...
268 # 1C0 tmp[0]
269
270 # 1B8 GT[7]
271 # ... ...
272 # 180 GT[0]
273
274 # 178 Reduce Data
275 # ... ...
276 # 0B8 Reduce Data
277 # 0B0 reserved
278 # 0A8 reserved
279 # 0A0 reserved
280 # 098 reserved
281 # 090 reserved
282 # 088 reduce result addr
283 # 080 exp[8]
284
285 # ...
286 # 048 exp[1]
287 # 040 exp[0]
288
289 # 038 reserved
290 # 030 loop_idx
291 # 028 pg
292 # 020 i
293 # 018 pData ; arg 4
294 # 010 pG ; arg 2
295 # 008 pResult ; arg 1
296 # 000 rsp ; stack pointer before subtract
297
298 my $rsp_offset = 0;
299 my $pResult_offset = 8*1 + $rsp_offset;
300 my $pG_offset = 8*1 + $pResult_offset;
301 my $pData_offset = 8*1 + $pG_offset;
302 my $i_offset = 8*1 + $pData_offset;
303 my $pg_offset = 8*1 + $i_offset;
304 my $loop_idx_offset = 8*1 + $pg_offset;
305 my $reserved1_offset = 8*1 + $loop_idx_offset;
306 my $exp_offset = 8*1 + $reserved1_offset;
307 my $red_result_addr_offset= 8*9 + $exp_offset;
308 my $reserved2_offset = 8*1 + $red_result_addr_offset;
309 my $Reduce_Data_offset = 8*5 + $reserved2_offset;
310 my $GT_offset = $Red_Data_Size + $Reduce_Data_offset;
311 my $tmp_offset = 8*8 + $GT_offset;
312 my $tmp16_offset = 8*8 + $tmp_offset;
313 my $garray_offset = 8*16 + $tmp16_offset;
314 my $mem_size = 8*8*32 + $garray_offset;
315
316 #
317 # Offsets within Reduce Data
318 #
319 #
320 # struct MODF_2FOLD_MONT_512_C1_DATA {
321 # UINT64 t[8][8];
322 # UINT64 m[8];
323 # UINT64 m1[8]; /* 2^768 % m */
324 # UINT64 m2[8]; /* 2^640 % m */
325 # UINT64 k1[2]; /* (- 1/m) % 2^128 */
326 # };
327
328 my $T = 0;
329 my $M = 512; # = 8 * 8 * 8
330 my $M1 = 576; # = 8 * 8 * 9 /* += 8 * 8 */
331 my $M2 = 640; # = 8 * 8 * 10 /* += 8 * 8 */
332 my $K1 = 704; # = 8 * 8 * 11 /* += 8 * 8 */
333
334 #
335 # FUNCTIONS
336 #
337
338 {{{
339 #
340 # MULADD_128x512 : Function to multiply 128-bits (2 qwords) by 512-bits (8 qwords)
341 # and add 512-bits (8 qwords)
342 # to get 640 bits (10 qwords)
343 # Input: 128-bit mul source: [rdi+8*1], rbp
344 # 512-bit mul source: [rsi+8*n]
345 # 512-bit add source: r15, r14, ..., r9, r8
346 # Output: r9, r8, r15, r14, r13, r12, r11, r10, [rcx+8*1], [rcx+8*0]
347 # Clobbers all regs except: rcx, rsi, rdi
348 $code.=<<___;
349 .type MULADD_128x512,\@abi-omnipotent
350 .align 16
351 MULADD_128x512:
352 ___
353 &MULSTEP_512([map("%r$_",(8..15))], "(+8*0)(%rcx)", "%rsi", "%rbp", "%rbx");
354 $code.=<<___;
355 mov (+8*1)(%rdi), %rbp
356 ___
357 &MULSTEP_512([map("%r$_",(9..15,8))], "(+8*1)(%rcx)", "%rsi", "%rbp", "%rbx");
358 $code.=<<___;
359 ret
360 .size MULADD_128x512,.-MULADD_128x512
361 ___
362 }}}
363
364 {{{
365 #MULADD_256x512 MACRO pDst, pA, pB, OP, TMP, X7, X6, X5, X4, X3, X2, X1, X0
366 #
367 # Inputs: pDst: Destination (768 bits, 12 qwords)
368 # pA: Multiplicand (1024 bits, 16 qwords)
369 # pB: Multiplicand (512 bits, 8 qwords)
370 # Dst = Ah * B + Al
371 # where Ah is (in qwords) A[15:12] (256 bits) and Al is A[7:0] (512 bits)
372 # Results in X3 X2 X1 X0 X7 X6 X5 X4 Dst[3:0]
373 # Uses registers: arguments, RAX, RDX
374 sub MULADD_256x512
375 {
376 my ($pDst, $pA, $pB, $OP, $TMP, $X)=@_;
377 $code.=<<___;
378 mov (+8*12)($pA), $OP
379 ___
380 &MULSTEP_512_ADD($X, "(+8*0)($pDst)", $pB, $pA, $OP, $TMP);
381 push(@$X,shift(@$X));
382
383 $code.=<<___;
384 mov (+8*13)($pA), $OP
385 ___
386 &MULSTEP_512($X, "(+8*1)($pDst)", $pB, $OP, $TMP);
387 push(@$X,shift(@$X));
388
389 $code.=<<___;
390 mov (+8*14)($pA), $OP
391 ___
392 &MULSTEP_512($X, "(+8*2)($pDst)", $pB, $OP, $TMP);
393 push(@$X,shift(@$X));
394
395 $code.=<<___;
396 mov (+8*15)($pA), $OP
397 ___
398 &MULSTEP_512($X, "(+8*3)($pDst)", $pB, $OP, $TMP);
399 push(@$X,shift(@$X));
400 }
401
402 #
403 # mont_reduce(UINT64 *x, /* 1024 bits, 16 qwords */
404 # UINT64 *m, /* 512 bits, 8 qwords */
405 # MODF_2FOLD_MONT_512_C1_DATA *data,
406 # UINT64 *r) /* 512 bits, 8 qwords */
407 # Input: x (number to be reduced): tmp16 (Implicit)
408 # m (modulus): [pM] (Implicit)
409 # data (reduce data): [pData] (Implicit)
410 # Output: r (result): Address in [red_res_addr]
411 # result also in: r9, r8, r15, r14, r13, r12, r11, r10
412
413 my @X=map("%r$_",(8..15));
414
415 $code.=<<___;
416 .type mont_reduce,\@abi-omnipotent
417 .align 16
418 mont_reduce:
419 ___
420
421 my $STACK_DEPTH = 8;
422 #
423 # X1 = Xh * M1 + Xl
424 $code.=<<___;
425 lea (+$Reduce_Data_offset+$X1_offset+$STACK_DEPTH)(%rsp), %rdi # pX1 (Dst) 769 bits, 13 qwords
426 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rsi # pM1 (Bsrc) 512 bits, 8 qwords
427 add \$$M1, %rsi
428 lea (+$tmp16_offset+$STACK_DEPTH)(%rsp), %rcx # X (Asrc) 1024 bits, 16 qwords
429
430 ___
431
432 &MULADD_256x512("%rdi", "%rcx", "%rsi", "%rbp", "%rbx", \@X); # rotates @X 4 times
433 # results in r11, r10, r9, r8, r15, r14, r13, r12, X1[3:0]
434
435 $code.=<<___;
436 xor %rax, %rax
437 # X1 += xl
438 add (+8*8)(%rcx), $X[4]
439 adc (+8*9)(%rcx), $X[5]
440 adc (+8*10)(%rcx), $X[6]
441 adc (+8*11)(%rcx), $X[7]
442 adc \$0, %rax
443 # X1 is now rax, r11-r8, r15-r12, tmp16[3:0]
444
445 #
446 # check for carry ;; carry stored in rax
447 mov $X[4], (+8*8)(%rdi) # rdi points to X1
448 mov $X[5], (+8*9)(%rdi)
449 mov $X[6], %rbp
450 mov $X[7], (+8*11)(%rdi)
451
452 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp)
453
454 mov (+8*0)(%rdi), $X[4]
455 mov (+8*1)(%rdi), $X[5]
456 mov (+8*2)(%rdi), $X[6]
457 mov (+8*3)(%rdi), $X[7]
458
459 # X1 is now stored in: X1[11], rbp, X1[9:8], r15-r8
460 # rdi -> X1
461 # rsi -> M1
462
463 #
464 # X2 = Xh * M2 + Xl
465 # do first part (X2 = Xh * M2)
466 add \$8*10, %rdi # rdi -> pXh ; 128 bits, 2 qwords
467 # Xh is actually { [rdi+8*1], rbp }
468 add \$`$M2-$M1`, %rsi # rsi -> M2
469 lea (+$Reduce_Data_offset+$X2_offset+$STACK_DEPTH)(%rsp), %rcx # rcx -> pX2 ; 641 bits, 11 qwords
470 ___
471 unshift(@X,pop(@X)); unshift(@X,pop(@X));
472 $code.=<<___;
473
474 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8
475 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0]
476 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rax
477
478 # X2 += Xl
479 add (+8*8-8*10)(%rdi), $X[6] # (-8*10) is to adjust rdi -> Xh to Xl
480 adc (+8*9-8*10)(%rdi), $X[7]
481 mov $X[6], (+8*8)(%rcx)
482 mov $X[7], (+8*9)(%rcx)
483
484 adc %rax, %rax
485 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp)
486
487 lea (+$Reduce_Data_offset+$Q_offset+$STACK_DEPTH)(%rsp), %rdi # rdi -> pQ ; 128 bits, 2 qwords
488 add \$`$K1-$M2`, %rsi # rsi -> pK1 ; 128 bits, 2 qwords
489
490 # MUL_128x128t128 rdi, rcx, rsi ; Q = X2 * K1 (bottom half)
491 # B1:B0 = rsi[1:0] = K1[1:0]
492 # A1:A0 = rcx[1:0] = X2[1:0]
493 # Result = rdi[1],rbp = Q[1],rbp
494 mov (%rsi), %r8 # B0
495 mov (+8*1)(%rsi), %rbx # B1
496
497 mov (%rcx), %rax # A0
498 mul %r8 # B0
499 mov %rax, %rbp
500 mov %rdx, %r9
501
502 mov (+8*1)(%rcx), %rax # A1
503 mul %r8 # B0
504 add %rax, %r9
505
506 mov (%rcx), %rax # A0
507 mul %rbx # B1
508 add %rax, %r9
509
510 mov %r9, (+8*1)(%rdi)
511 # end MUL_128x128t128
512
513 sub \$`$K1-$M`, %rsi
514
515 mov (%rcx), $X[6]
516 mov (+8*1)(%rcx), $X[7] # r9:r8 = X2[1:0]
517
518 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8
519 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0]
520
521 # load first half of m to rdx, rdi, rbx, rax
522 # moved this here for efficiency
523 mov (+8*0)(%rsi), %rax
524 mov (+8*1)(%rsi), %rbx
525 mov (+8*2)(%rsi), %rdi
526 mov (+8*3)(%rsi), %rdx
527
528 # continue with reduction
529 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rbp
530
531 add (+8*8)(%rcx), $X[6]
532 adc (+8*9)(%rcx), $X[7]
533
534 #accumulate the final carry to rbp
535 adc %rbp, %rbp
536
537 # Add in overflow corrections: R = (X2>>128) += T[overflow]
538 # R = {r9, r8, r15, r14, ..., r10}
539 shl \$3, %rbp
540 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rcx # rsi -> Data (and points to T)
541 add %rcx, %rbp # pT ; 512 bits, 8 qwords, spread out
542
543 # rsi will be used to generate a mask after the addition
544 xor %rsi, %rsi
545
546 add (+8*8*0)(%rbp), $X[0]
547 adc (+8*8*1)(%rbp), $X[1]
548 adc (+8*8*2)(%rbp), $X[2]
549 adc (+8*8*3)(%rbp), $X[3]
550 adc (+8*8*4)(%rbp), $X[4]
551 adc (+8*8*5)(%rbp), $X[5]
552 adc (+8*8*6)(%rbp), $X[6]
553 adc (+8*8*7)(%rbp), $X[7]
554
555 # if there is a carry: rsi = 0xFFFFFFFFFFFFFFFF
556 # if carry is clear: rsi = 0x0000000000000000
557 sbb \$0, %rsi
558
559 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m
560 and %rsi, %rax
561 and %rsi, %rbx
562 and %rsi, %rdi
563 and %rsi, %rdx
564
565 mov \$1, %rbp
566 sub %rax, $X[0]
567 sbb %rbx, $X[1]
568 sbb %rdi, $X[2]
569 sbb %rdx, $X[3]
570
571 # if there is a borrow: rbp = 0
572 # if there is no borrow: rbp = 1
573 # this is used to save the borrows in between the first half and the 2nd half of the subtraction of m
574 sbb \$0, %rbp
575
576 #load second half of m to rdx, rdi, rbx, rax
577
578 add \$$M, %rcx
579 mov (+8*4)(%rcx), %rax
580 mov (+8*5)(%rcx), %rbx
581 mov (+8*6)(%rcx), %rdi
582 mov (+8*7)(%rcx), %rdx
583
584 # use the rsi mask as before
585 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m
586 and %rsi, %rax
587 and %rsi, %rbx
588 and %rsi, %rdi
589 and %rsi, %rdx
590
591 # if rbp = 0, there was a borrow before, it is moved to the carry flag
592 # if rbp = 1, there was not a borrow before, carry flag is cleared
593 sub \$1, %rbp
594
595 sbb %rax, $X[4]
596 sbb %rbx, $X[5]
597 sbb %rdi, $X[6]
598 sbb %rdx, $X[7]
599
600 # write R back to memory
601
602 mov (+$red_result_addr_offset+$STACK_DEPTH)(%rsp), %rsi
603 mov $X[0], (+8*0)(%rsi)
604 mov $X[1], (+8*1)(%rsi)
605 mov $X[2], (+8*2)(%rsi)
606 mov $X[3], (+8*3)(%rsi)
607 mov $X[4], (+8*4)(%rsi)
608 mov $X[5], (+8*5)(%rsi)
609 mov $X[6], (+8*6)(%rsi)
610 mov $X[7], (+8*7)(%rsi)
611
612 ret
613 .size mont_reduce,.-mont_reduce
614 ___
615 }}}
616
617 {{{
618 #MUL_512x512 MACRO pDst, pA, pB, x7, x6, x5, x4, x3, x2, x1, x0, tmp*2
619 #
620 # Inputs: pDst: Destination (1024 bits, 16 qwords)
621 # pA: Multiplicand (512 bits, 8 qwords)
622 # pB: Multiplicand (512 bits, 8 qwords)
623 # Uses registers rax, rdx, args
624 # B operand in [pB] and also in x7...x0
625 sub MUL_512x512
626 {
627 my ($pDst, $pA, $pB, $x, $OP, $TMP, $pDst_o)=@_;
628 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/);
629 my @X=@$x; # make a copy
630
631 $code.=<<___;
632 mov (+8*0)($pA), $OP
633
634 mov $X[0], %rax
635 mul $OP # rdx:rax = %OP * [0]
636 mov %rax, (+$pDst_o+8*0)($pDst)
637 mov %rdx, $X[0]
638 ___
639 for(my $i=1;$i<8;$i++) {
640 $code.=<<___;
641 mov $X[$i], %rax
642 mul $OP # rdx:rax = %OP * [$i]
643 add %rax, $X[$i-1]
644 adc \$0, %rdx
645 mov %rdx, $X[$i]
646 ___
647 }
648
649 for(my $i=1;$i<8;$i++) {
650 $code.=<<___;
651 mov (+8*$i)($pA), $OP
652 ___
653
654 &MULSTEP_512(\@X, "(+$pDst_o+8*$i)($pDst)", $pB, $OP, $TMP);
655 push(@X,shift(@X));
656 }
657
658 $code.=<<___;
659 mov $X[0], (+$pDst_o+8*8)($pDst)
660 mov $X[1], (+$pDst_o+8*9)($pDst)
661 mov $X[2], (+$pDst_o+8*10)($pDst)
662 mov $X[3], (+$pDst_o+8*11)($pDst)
663 mov $X[4], (+$pDst_o+8*12)($pDst)
664 mov $X[5], (+$pDst_o+8*13)($pDst)
665 mov $X[6], (+$pDst_o+8*14)($pDst)
666 mov $X[7], (+$pDst_o+8*15)($pDst)
667 ___
668 }
669
670 #
671 # mont_mul_a3b : subroutine to compute (Src1 * Src2) % M (all 512-bits)
672 # Input: src1: Address of source 1: rdi
673 # src2: Address of source 2: rsi
674 # Output: dst: Address of destination: [red_res_addr]
675 # src2 and result also in: r9, r8, r15, r14, r13, r12, r11, r10
676 # Temp: Clobbers [tmp16], all registers
677 $code.=<<___;
678 .type mont_mul_a3b,\@abi-omnipotent
679 .align 16
680 mont_mul_a3b:
681 #
682 # multiply tmp = src1 * src2
683 # For multiply: dst = rcx, src1 = rdi, src2 = rsi
684 # stack depth is extra 8 from call
685 ___
686 &MUL_512x512("%rsp+$tmp16_offset+8", "%rdi", "%rsi", [map("%r$_",(10..15,8..9))], "%rbp", "%rbx");
687 $code.=<<___;
688 #
689 # Dst = tmp % m
690 # Call reduce(tmp, m, data, dst)
691
692 # tail recursion optimization: jmp to mont_reduce and return from there
693 jmp mont_reduce
694 # call mont_reduce
695 # ret
696 .size mont_mul_a3b,.-mont_mul_a3b
697 ___
698 }}}
699
700 {{{
701 #SQR_512 MACRO pDest, pA, x7, x6, x5, x4, x3, x2, x1, x0, tmp*4
702 #
703 # Input in memory [pA] and also in x7...x0
704 # Uses all argument registers plus rax and rdx
705 #
706 # This version computes all of the off-diagonal terms into memory,
707 # and then it adds in the diagonal terms
708
709 sub SQR_512
710 {
711 my ($pDst, $pA, $x, $A, $tmp, $x7, $x6, $pDst_o)=@_;
712 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/);
713 my @X=@$x; # make a copy
714 $code.=<<___;
715 # ------------------
716 # first pass 01...07
717 # ------------------
718 mov $X[0], $A
719
720 mov $X[1],%rax
721 mul $A
722 mov %rax, (+$pDst_o+8*1)($pDst)
723 ___
724 for(my $i=2;$i<8;$i++) {
725 $code.=<<___;
726 mov %rdx, $X[$i-2]
727 mov $X[$i],%rax
728 mul $A
729 add %rax, $X[$i-2]
730 adc \$0, %rdx
731 ___
732 }
733 $code.=<<___;
734 mov %rdx, $x7
735
736 mov $X[0], (+$pDst_o+8*2)($pDst)
737
738 # ------------------
739 # second pass 12...17
740 # ------------------
741
742 mov (+8*1)($pA), $A
743
744 mov (+8*2)($pA),%rax
745 mul $A
746 add %rax, $X[1]
747 adc \$0, %rdx
748 mov $X[1], (+$pDst_o+8*3)($pDst)
749
750 mov %rdx, $X[0]
751 mov (+8*3)($pA),%rax
752 mul $A
753 add %rax, $X[2]
754 adc \$0, %rdx
755 add $X[0], $X[2]
756 adc \$0, %rdx
757 mov $X[2], (+$pDst_o+8*4)($pDst)
758
759 mov %rdx, $X[0]
760 mov (+8*4)($pA),%rax
761 mul $A
762 add %rax, $X[3]
763 adc \$0, %rdx
764 add $X[0], $X[3]
765 adc \$0, %rdx
766
767 mov %rdx, $X[0]
768 mov (+8*5)($pA),%rax
769 mul $A
770 add %rax, $X[4]
771 adc \$0, %rdx
772 add $X[0], $X[4]
773 adc \$0, %rdx
774
775 mov %rdx, $X[0]
776 mov $X[6],%rax
777 mul $A
778 add %rax, $X[5]
779 adc \$0, %rdx
780 add $X[0], $X[5]
781 adc \$0, %rdx
782
783 mov %rdx, $X[0]
784 mov $X[7],%rax
785 mul $A
786 add %rax, $x7
787 adc \$0, %rdx
788 add $X[0], $x7
789 adc \$0, %rdx
790
791 mov %rdx, $X[1]
792
793 # ------------------
794 # third pass 23...27
795 # ------------------
796 mov (+8*2)($pA), $A
797
798 mov (+8*3)($pA),%rax
799 mul $A
800 add %rax, $X[3]
801 adc \$0, %rdx
802 mov $X[3], (+$pDst_o+8*5)($pDst)
803
804 mov %rdx, $X[0]
805 mov (+8*4)($pA),%rax
806 mul $A
807 add %rax, $X[4]
808 adc \$0, %rdx
809 add $X[0], $X[4]
810 adc \$0, %rdx
811 mov $X[4], (+$pDst_o+8*6)($pDst)
812
813 mov %rdx, $X[0]
814 mov (+8*5)($pA),%rax
815 mul $A
816 add %rax, $X[5]
817 adc \$0, %rdx
818 add $X[0], $X[5]
819 adc \$0, %rdx
820
821 mov %rdx, $X[0]
822 mov $X[6],%rax
823 mul $A
824 add %rax, $x7
825 adc \$0, %rdx
826 add $X[0], $x7
827 adc \$0, %rdx
828
829 mov %rdx, $X[0]
830 mov $X[7],%rax
831 mul $A
832 add %rax, $X[1]
833 adc \$0, %rdx
834 add $X[0], $X[1]
835 adc \$0, %rdx
836
837 mov %rdx, $X[2]
838
839 # ------------------
840 # fourth pass 34...37
841 # ------------------
842
843 mov (+8*3)($pA), $A
844
845 mov (+8*4)($pA),%rax
846 mul $A
847 add %rax, $X[5]
848 adc \$0, %rdx
849 mov $X[5], (+$pDst_o+8*7)($pDst)
850
851 mov %rdx, $X[0]
852 mov (+8*5)($pA),%rax
853 mul $A
854 add %rax, $x7
855 adc \$0, %rdx
856 add $X[0], $x7
857 adc \$0, %rdx
858 mov $x7, (+$pDst_o+8*8)($pDst)
859
860 mov %rdx, $X[0]
861 mov $X[6],%rax
862 mul $A
863 add %rax, $X[1]
864 adc \$0, %rdx
865 add $X[0], $X[1]
866 adc \$0, %rdx
867
868 mov %rdx, $X[0]
869 mov $X[7],%rax
870 mul $A
871 add %rax, $X[2]
872 adc \$0, %rdx
873 add $X[0], $X[2]
874 adc \$0, %rdx
875
876 mov %rdx, $X[5]
877
878 # ------------------
879 # fifth pass 45...47
880 # ------------------
881 mov (+8*4)($pA), $A
882
883 mov (+8*5)($pA),%rax
884 mul $A
885 add %rax, $X[1]
886 adc \$0, %rdx
887 mov $X[1], (+$pDst_o+8*9)($pDst)
888
889 mov %rdx, $X[0]
890 mov $X[6],%rax
891 mul $A
892 add %rax, $X[2]
893 adc \$0, %rdx
894 add $X[0], $X[2]
895 adc \$0, %rdx
896 mov $X[2], (+$pDst_o+8*10)($pDst)
897
898 mov %rdx, $X[0]
899 mov $X[7],%rax
900 mul $A
901 add %rax, $X[5]
902 adc \$0, %rdx
903 add $X[0], $X[5]
904 adc \$0, %rdx
905
906 mov %rdx, $X[1]
907
908 # ------------------
909 # sixth pass 56...57
910 # ------------------
911 mov (+8*5)($pA), $A
912
913 mov $X[6],%rax
914 mul $A
915 add %rax, $X[5]
916 adc \$0, %rdx
917 mov $X[5], (+$pDst_o+8*11)($pDst)
918
919 mov %rdx, $X[0]
920 mov $X[7],%rax
921 mul $A
922 add %rax, $X[1]
923 adc \$0, %rdx
924 add $X[0], $X[1]
925 adc \$0, %rdx
926 mov $X[1], (+$pDst_o+8*12)($pDst)
927
928 mov %rdx, $X[2]
929
930 # ------------------
931 # seventh pass 67
932 # ------------------
933 mov $X[6], $A
934
935 mov $X[7],%rax
936 mul $A
937 add %rax, $X[2]
938 adc \$0, %rdx
939 mov $X[2], (+$pDst_o+8*13)($pDst)
940
941 mov %rdx, (+$pDst_o+8*14)($pDst)
942
943 # start finalize (add in squares, and double off-terms)
944 mov (+$pDst_o+8*1)($pDst), $X[0]
945 mov (+$pDst_o+8*2)($pDst), $X[1]
946 mov (+$pDst_o+8*3)($pDst), $X[2]
947 mov (+$pDst_o+8*4)($pDst), $X[3]
948 mov (+$pDst_o+8*5)($pDst), $X[4]
949 mov (+$pDst_o+8*6)($pDst), $X[5]
950
951 mov (+8*3)($pA), %rax
952 mul %rax
953 mov %rax, $x6
954 mov %rdx, $X[6]
955
956 add $X[0], $X[0]
957 adc $X[1], $X[1]
958 adc $X[2], $X[2]
959 adc $X[3], $X[3]
960 adc $X[4], $X[4]
961 adc $X[5], $X[5]
962 adc \$0, $X[6]
963
964 mov (+8*0)($pA), %rax
965 mul %rax
966 mov %rax, (+$pDst_o+8*0)($pDst)
967 mov %rdx, $A
968
969 mov (+8*1)($pA), %rax
970 mul %rax
971
972 add $A, $X[0]
973 adc %rax, $X[1]
974 adc \$0, %rdx
975
976 mov %rdx, $A
977 mov $X[0], (+$pDst_o+8*1)($pDst)
978 mov $X[1], (+$pDst_o+8*2)($pDst)
979
980 mov (+8*2)($pA), %rax
981 mul %rax
982
983 add $A, $X[2]
984 adc %rax, $X[3]
985 adc \$0, %rdx
986
987 mov %rdx, $A
988
989 mov $X[2], (+$pDst_o+8*3)($pDst)
990 mov $X[3], (+$pDst_o+8*4)($pDst)
991
992 xor $tmp, $tmp
993 add $A, $X[4]
994 adc $x6, $X[5]
995 adc \$0, $tmp
996
997 mov $X[4], (+$pDst_o+8*5)($pDst)
998 mov $X[5], (+$pDst_o+8*6)($pDst)
999
1000 # %%tmp has 0/1 in column 7
1001 # %%A6 has a full value in column 7
1002
1003 mov (+$pDst_o+8*7)($pDst), $X[0]
1004 mov (+$pDst_o+8*8)($pDst), $X[1]
1005 mov (+$pDst_o+8*9)($pDst), $X[2]
1006 mov (+$pDst_o+8*10)($pDst), $X[3]
1007 mov (+$pDst_o+8*11)($pDst), $X[4]
1008 mov (+$pDst_o+8*12)($pDst), $X[5]
1009 mov (+$pDst_o+8*13)($pDst), $x6
1010 mov (+$pDst_o+8*14)($pDst), $x7
1011
1012 mov $X[7], %rax
1013 mul %rax
1014 mov %rax, $X[7]
1015 mov %rdx, $A
1016
1017 add $X[0], $X[0]
1018 adc $X[1], $X[1]
1019 adc $X[2], $X[2]
1020 adc $X[3], $X[3]
1021 adc $X[4], $X[4]
1022 adc $X[5], $X[5]
1023 adc $x6, $x6
1024 adc $x7, $x7
1025 adc \$0, $A
1026
1027 add $tmp, $X[0]
1028
1029 mov (+8*4)($pA), %rax
1030 mul %rax
1031
1032 add $X[6], $X[0]
1033 adc %rax, $X[1]
1034 adc \$0, %rdx
1035
1036 mov %rdx, $tmp
1037
1038 mov $X[0], (+$pDst_o+8*7)($pDst)
1039 mov $X[1], (+$pDst_o+8*8)($pDst)
1040
1041 mov (+8*5)($pA), %rax
1042 mul %rax
1043
1044 add $tmp, $X[2]
1045 adc %rax, $X[3]
1046 adc \$0, %rdx
1047
1048 mov %rdx, $tmp
1049
1050 mov $X[2], (+$pDst_o+8*9)($pDst)
1051 mov $X[3], (+$pDst_o+8*10)($pDst)
1052
1053 mov (+8*6)($pA), %rax
1054 mul %rax
1055
1056 add $tmp, $X[4]
1057 adc %rax, $X[5]
1058 adc \$0, %rdx
1059
1060 mov $X[4], (+$pDst_o+8*11)($pDst)
1061 mov $X[5], (+$pDst_o+8*12)($pDst)
1062
1063 add %rdx, $x6
1064 adc $X[7], $x7
1065 adc \$0, $A
1066
1067 mov $x6, (+$pDst_o+8*13)($pDst)
1068 mov $x7, (+$pDst_o+8*14)($pDst)
1069 mov $A, (+$pDst_o+8*15)($pDst)
1070 ___
1071 }
1072
1073 #
1074 # sqr_reduce: subroutine to compute Result = reduce(Result * Result)
1075 #
1076 # input and result also in: r9, r8, r15, r14, r13, r12, r11, r10
1077 #
1078 $code.=<<___;
1079 .type sqr_reduce,\@abi-omnipotent
1080 .align 16
1081 sqr_reduce:
1082 mov (+$pResult_offset+8)(%rsp), %rcx
1083 ___
1084 &SQR_512("%rsp+$tmp16_offset+8", "%rcx", [map("%r$_",(10..15,8..9))], "%rbx", "%rbp", "%rsi", "%rdi");
1085 $code.=<<___;
1086 # tail recursion optimization: jmp to mont_reduce and return from there
1087 jmp mont_reduce
1088 # call mont_reduce
1089 # ret
1090 .size sqr_reduce,.-sqr_reduce
1091 ___
1092 }}}
1093
1094 #
1095 # MAIN FUNCTION
1096 #
1097
1098 #mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */
1099 # UINT64 *g, /* 512 bits, 8 qwords */
1100 # UINT64 *exp, /* 512 bits, 8 qwords */
1101 # struct mod_ctx_512 *data)
1102
1103 # window size = 5
1104 # table size = 2^5 = 32
1105 #table_entries equ 32
1106 #table_size equ table_entries * 8
1107 $code.=<<___;
1108 .globl mod_exp_512
1109 .type mod_exp_512,\@function,4
1110 mod_exp_512:
1111 push %rbp
1112 push %rbx
1113 push %r12
1114 push %r13
1115 push %r14
1116 push %r15
1117
1118 # adjust stack down and then align it with cache boundary
1119 mov %rsp, %r8
1120 sub \$$mem_size, %rsp
1121 and \$-64, %rsp
1122
1123 # store previous stack pointer and arguments
1124 mov %r8, (+$rsp_offset)(%rsp)
1125 mov %rdi, (+$pResult_offset)(%rsp)
1126 mov %rsi, (+$pG_offset)(%rsp)
1127 mov %rcx, (+$pData_offset)(%rsp)
1128 .Lbody:
1129 # transform g into montgomery space
1130 # GT = reduce(g * C2) = reduce(g * (2^256))
1131 # reduce expects to have the input in [tmp16]
1132 pxor %xmm4, %xmm4
1133 movdqu (+16*0)(%rsi), %xmm0
1134 movdqu (+16*1)(%rsi), %xmm1
1135 movdqu (+16*2)(%rsi), %xmm2
1136 movdqu (+16*3)(%rsi), %xmm3
1137 movdqa %xmm4, (+$tmp16_offset+16*0)(%rsp)
1138 movdqa %xmm4, (+$tmp16_offset+16*1)(%rsp)
1139 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp)
1140 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp)
1141 movdqa %xmm0, (+$tmp16_offset+16*2)(%rsp)
1142 movdqa %xmm1, (+$tmp16_offset+16*3)(%rsp)
1143 movdqa %xmm2, (+$tmp16_offset+16*4)(%rsp)
1144 movdqa %xmm3, (+$tmp16_offset+16*5)(%rsp)
1145
1146 # load pExp before rdx gets blown away
1147 movdqu (+16*0)(%rdx), %xmm0
1148 movdqu (+16*1)(%rdx), %xmm1
1149 movdqu (+16*2)(%rdx), %xmm2
1150 movdqu (+16*3)(%rdx), %xmm3
1151
1152 lea (+$GT_offset)(%rsp), %rbx
1153 mov %rbx, (+$red_result_addr_offset)(%rsp)
1154 call mont_reduce
1155
1156 # Initialize tmp = C
1157 lea (+$tmp_offset)(%rsp), %rcx
1158 xor %rax, %rax
1159 mov %rax, (+8*0)(%rcx)
1160 mov %rax, (+8*1)(%rcx)
1161 mov %rax, (+8*3)(%rcx)
1162 mov %rax, (+8*4)(%rcx)
1163 mov %rax, (+8*5)(%rcx)
1164 mov %rax, (+8*6)(%rcx)
1165 mov %rax, (+8*7)(%rcx)
1166 mov %rax, (+$exp_offset+8*8)(%rsp)
1167 movq \$1, (+8*2)(%rcx)
1168
1169 lea (+$garray_offset)(%rsp), %rbp
1170 mov %rcx, %rsi # pTmp
1171 mov %rbp, %rdi # Garray[][0]
1172 ___
1173
1174 &swizzle("%rdi", "%rcx", "%rax", "%rbx");
1175
1176 # for (rax = 31; rax != 0; rax--) {
1177 # tmp = reduce(tmp * G)
1178 # swizzle(pg, tmp);
1179 # pg += 2; }
1180 $code.=<<___;
1181 mov \$31, %rax
1182 mov %rax, (+$i_offset)(%rsp)
1183 mov %rbp, (+$pg_offset)(%rsp)
1184 # rsi -> pTmp
1185 mov %rsi, (+$red_result_addr_offset)(%rsp)
1186 mov (+8*0)(%rsi), %r10
1187 mov (+8*1)(%rsi), %r11
1188 mov (+8*2)(%rsi), %r12
1189 mov (+8*3)(%rsi), %r13
1190 mov (+8*4)(%rsi), %r14
1191 mov (+8*5)(%rsi), %r15
1192 mov (+8*6)(%rsi), %r8
1193 mov (+8*7)(%rsi), %r9
1194 init_loop:
1195 lea (+$GT_offset)(%rsp), %rdi
1196 call mont_mul_a3b
1197 lea (+$tmp_offset)(%rsp), %rsi
1198 mov (+$pg_offset)(%rsp), %rbp
1199 add \$2, %rbp
1200 mov %rbp, (+$pg_offset)(%rsp)
1201 mov %rsi, %rcx # rcx = rsi = addr of tmp
1202 ___
1203
1204 &swizzle("%rbp", "%rcx", "%rax", "%rbx");
1205 $code.=<<___;
1206 mov (+$i_offset)(%rsp), %rax
1207 sub \$1, %rax
1208 mov %rax, (+$i_offset)(%rsp)
1209 jne init_loop
1210
1211 #
1212 # Copy exponent onto stack
1213 movdqa %xmm0, (+$exp_offset+16*0)(%rsp)
1214 movdqa %xmm1, (+$exp_offset+16*1)(%rsp)
1215 movdqa %xmm2, (+$exp_offset+16*2)(%rsp)
1216 movdqa %xmm3, (+$exp_offset+16*3)(%rsp)
1217
1218
1219 #
1220 # Do exponentiation
1221 # Initialize result to G[exp{511:507}]
1222 mov (+$exp_offset+62)(%rsp), %eax
1223 mov %rax, %rdx
1224 shr \$11, %rax
1225 and \$0x07FF, %edx
1226 mov %edx, (+$exp_offset+62)(%rsp)
1227 lea (+$garray_offset)(%rsp,%rax,2), %rsi
1228 mov (+$pResult_offset)(%rsp), %rdx
1229 ___
1230
1231 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax");
1232
1233 #
1234 # Loop variables
1235 # rcx = [loop_idx] = index: 510-5 to 0 by 5
1236 $code.=<<___;
1237 movq \$505, (+$loop_idx_offset)(%rsp)
1238
1239 mov (+$pResult_offset)(%rsp), %rcx
1240 mov %rcx, (+$red_result_addr_offset)(%rsp)
1241 mov (+8*0)(%rcx), %r10
1242 mov (+8*1)(%rcx), %r11
1243 mov (+8*2)(%rcx), %r12
1244 mov (+8*3)(%rcx), %r13
1245 mov (+8*4)(%rcx), %r14
1246 mov (+8*5)(%rcx), %r15
1247 mov (+8*6)(%rcx), %r8
1248 mov (+8*7)(%rcx), %r9
1249 jmp sqr_2
1250
1251 main_loop_a3b:
1252 call sqr_reduce
1253 call sqr_reduce
1254 call sqr_reduce
1255 sqr_2:
1256 call sqr_reduce
1257 call sqr_reduce
1258
1259 #
1260 # Do multiply, first look up proper value in Garray
1261 mov (+$loop_idx_offset)(%rsp), %rcx # bit index
1262 mov %rcx, %rax
1263 shr \$4, %rax # rax is word pointer
1264 mov (+$exp_offset)(%rsp,%rax,2), %edx
1265 and \$15, %rcx
1266 shrq %cl, %rdx
1267 and \$0x1F, %rdx
1268
1269 lea (+$garray_offset)(%rsp,%rdx,2), %rsi
1270 lea (+$tmp_offset)(%rsp), %rdx
1271 mov %rdx, %rdi
1272 ___
1273
1274 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax");
1275 # rdi = tmp = pG
1276
1277 #
1278 # Call mod_mul_a1(pDst, pSrc1, pSrc2, pM, pData)
1279 # result result pG M Data
1280 $code.=<<___;
1281 mov (+$pResult_offset)(%rsp), %rsi
1282 call mont_mul_a3b
1283
1284 #
1285 # finish loop
1286 mov (+$loop_idx_offset)(%rsp), %rcx
1287 sub \$5, %rcx
1288 mov %rcx, (+$loop_idx_offset)(%rsp)
1289 jge main_loop_a3b
1290
1291 #
1292
1293 end_main_loop_a3b:
1294 # transform result out of Montgomery space
1295 # result = reduce(result)
1296 mov (+$pResult_offset)(%rsp), %rdx
1297 pxor %xmm4, %xmm4
1298 movdqu (+16*0)(%rdx), %xmm0
1299 movdqu (+16*1)(%rdx), %xmm1
1300 movdqu (+16*2)(%rdx), %xmm2
1301 movdqu (+16*3)(%rdx), %xmm3
1302 movdqa %xmm4, (+$tmp16_offset+16*4)(%rsp)
1303 movdqa %xmm4, (+$tmp16_offset+16*5)(%rsp)
1304 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp)
1305 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp)
1306 movdqa %xmm0, (+$tmp16_offset+16*0)(%rsp)
1307 movdqa %xmm1, (+$tmp16_offset+16*1)(%rsp)
1308 movdqa %xmm2, (+$tmp16_offset+16*2)(%rsp)
1309 movdqa %xmm3, (+$tmp16_offset+16*3)(%rsp)
1310 call mont_reduce
1311
1312 # If result > m, subract m
1313 # load result into r15:r8
1314 mov (+$pResult_offset)(%rsp), %rax
1315 mov (+8*0)(%rax), %r8
1316 mov (+8*1)(%rax), %r9
1317 mov (+8*2)(%rax), %r10
1318 mov (+8*3)(%rax), %r11
1319 mov (+8*4)(%rax), %r12
1320 mov (+8*5)(%rax), %r13
1321 mov (+8*6)(%rax), %r14
1322 mov (+8*7)(%rax), %r15
1323
1324 # subtract m
1325 mov (+$pData_offset)(%rsp), %rbx
1326 add \$$M, %rbx
1327
1328 sub (+8*0)(%rbx), %r8
1329 sbb (+8*1)(%rbx), %r9
1330 sbb (+8*2)(%rbx), %r10
1331 sbb (+8*3)(%rbx), %r11
1332 sbb (+8*4)(%rbx), %r12
1333 sbb (+8*5)(%rbx), %r13
1334 sbb (+8*6)(%rbx), %r14
1335 sbb (+8*7)(%rbx), %r15
1336
1337 # if Carry is clear, replace result with difference
1338 mov (+8*0)(%rax), %rsi
1339 mov (+8*1)(%rax), %rdi
1340 mov (+8*2)(%rax), %rcx
1341 mov (+8*3)(%rax), %rdx
1342 cmovnc %r8, %rsi
1343 cmovnc %r9, %rdi
1344 cmovnc %r10, %rcx
1345 cmovnc %r11, %rdx
1346 mov %rsi, (+8*0)(%rax)
1347 mov %rdi, (+8*1)(%rax)
1348 mov %rcx, (+8*2)(%rax)
1349 mov %rdx, (+8*3)(%rax)
1350
1351 mov (+8*4)(%rax), %rsi
1352 mov (+8*5)(%rax), %rdi
1353 mov (+8*6)(%rax), %rcx
1354 mov (+8*7)(%rax), %rdx
1355 cmovnc %r12, %rsi
1356 cmovnc %r13, %rdi
1357 cmovnc %r14, %rcx
1358 cmovnc %r15, %rdx
1359 mov %rsi, (+8*4)(%rax)
1360 mov %rdi, (+8*5)(%rax)
1361 mov %rcx, (+8*6)(%rax)
1362 mov %rdx, (+8*7)(%rax)
1363
1364 mov (+$rsp_offset)(%rsp), %rsi
1365 mov 0(%rsi),%r15
1366 mov 8(%rsi),%r14
1367 mov 16(%rsi),%r13
1368 mov 24(%rsi),%r12
1369 mov 32(%rsi),%rbx
1370 mov 40(%rsi),%rbp
1371 lea 48(%rsi),%rsp
1372 .Lepilogue:
1373 ret
1374 .size mod_exp_512, . - mod_exp_512
1375 ___
1376
1377 if ($win64) {
1378 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1379 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
1380 my $rec="%rcx";
1381 my $frame="%rdx";
1382 my $context="%r8";
1383 my $disp="%r9";
1384
1385 $code.=<<___;
1386 .extern __imp_RtlVirtualUnwind
1387 .type mod_exp_512_se_handler,\@abi-omnipotent
1388 .align 16
1389 mod_exp_512_se_handler:
1390 push %rsi
1391 push %rdi
1392 push %rbx
1393 push %rbp
1394 push %r12
1395 push %r13
1396 push %r14
1397 push %r15
1398 pushfq
1399 sub \$64,%rsp
1400
1401 mov 120($context),%rax # pull context->Rax
1402 mov 248($context),%rbx # pull context->Rip
1403
1404 lea .Lbody(%rip),%r10
1405 cmp %r10,%rbx # context->Rip<prologue label
1406 jb .Lin_prologue
1407
1408 mov 152($context),%rax # pull context->Rsp
1409
1410 lea .Lepilogue(%rip),%r10
1411 cmp %r10,%rbx # context->Rip>=epilogue label
1412 jae .Lin_prologue
1413
1414 mov $rsp_offset(%rax),%rax # pull saved Rsp
1415
1416 mov 32(%rax),%rbx
1417 mov 40(%rax),%rbp
1418 mov 24(%rax),%r12
1419 mov 16(%rax),%r13
1420 mov 8(%rax),%r14
1421 mov 0(%rax),%r15
1422 lea 48(%rax),%rax
1423 mov %rbx,144($context) # restore context->Rbx
1424 mov %rbp,160($context) # restore context->Rbp
1425 mov %r12,216($context) # restore context->R12
1426 mov %r13,224($context) # restore context->R13
1427 mov %r14,232($context) # restore context->R14
1428 mov %r15,240($context) # restore context->R15
1429
1430 .Lin_prologue:
1431 mov 8(%rax),%rdi
1432 mov 16(%rax),%rsi
1433 mov %rax,152($context) # restore context->Rsp
1434 mov %rsi,168($context) # restore context->Rsi
1435 mov %rdi,176($context) # restore context->Rdi
1436
1437 mov 40($disp),%rdi # disp->ContextRecord
1438 mov $context,%rsi # context
1439 mov \$154,%ecx # sizeof(CONTEXT)
1440 .long 0xa548f3fc # cld; rep movsq
1441
1442 mov $disp,%rsi
1443 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
1444 mov 8(%rsi),%rdx # arg2, disp->ImageBase
1445 mov 0(%rsi),%r8 # arg3, disp->ControlPc
1446 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
1447 mov 40(%rsi),%r10 # disp->ContextRecord
1448 lea 56(%rsi),%r11 # &disp->HandlerData
1449 lea 24(%rsi),%r12 # &disp->EstablisherFrame
1450 mov %r10,32(%rsp) # arg5
1451 mov %r11,40(%rsp) # arg6
1452 mov %r12,48(%rsp) # arg7
1453 mov %rcx,56(%rsp) # arg8, (NULL)
1454 call *__imp_RtlVirtualUnwind(%rip)
1455
1456 mov \$1,%eax # ExceptionContinueSearch
1457 add \$64,%rsp
1458 popfq
1459 pop %r15
1460 pop %r14
1461 pop %r13
1462 pop %r12
1463 pop %rbp
1464 pop %rbx
1465 pop %rdi
1466 pop %rsi
1467 ret
1468 .size mod_exp_512_se_handler,.-mod_exp_512_se_handler
1469
1470 .section .pdata
1471 .align 4
1472 .rva .LSEH_begin_mod_exp_512
1473 .rva .LSEH_end_mod_exp_512
1474 .rva .LSEH_info_mod_exp_512
1475
1476 .section .xdata
1477 .align 8
1478 .LSEH_info_mod_exp_512:
1479 .byte 9,0,0,0
1480 .rva mod_exp_512_se_handler
1481 ___
1482 }
1483
1484 sub reg_part {
1485 my ($reg,$conv)=@_;
1486 if ($reg =~ /%r[0-9]+/) { $reg .= $conv; }
1487 elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; }
1488 elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; }
1489 elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; }
1490 return $reg;
1491 }
1492
1493 $code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem;
1494 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1495 $code =~ s/(\(\+[^)]+\))/eval $1/gem;
1496 print $code;
1497 close STDOUT;