source/libs/gmp/gmp-src/mpn/x86_64/coreihwl/aorsmul_1.asm

   1 dnl  AMD64 mpn_addmul_1 and mpn_submul_1 optimised for Intel Haswell.
   2
   3 dnl  Contributed to the GNU project by Torbjörn Granlund.
   4
   5 dnl  Copyright 2013 Free Software Foundation, Inc.
   6
   7 dnl  This file is part of the GNU MP Library.
   8 dnl
   9 dnl  The GNU MP Library is free software; you can redistribute it and/or modify
  10 dnl  it under the terms of either:
  11 dnl
  12 dnl    * the GNU Lesser General Public License as published by the Free
  13 dnl      Software Foundation; either version 3 of the License, or (at your
  14 dnl      option) any later version.
  15 dnl
  16 dnl  or
  17 dnl
  18 dnl    * the GNU General Public License as published by the Free Software
  19 dnl      Foundation; either version 2 of the License, or (at your option) any
  20 dnl      later version.
  21 dnl
  22 dnl  or both in parallel, as here.
  23 dnl
  24 dnl  The GNU MP Library is distributed in the hope that it will be useful, but
  25 dnl  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  26 dnl  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  27 dnl  for more details.
  28 dnl
  29 dnl  You should have received copies of the GNU General Public License and the
  30 dnl  GNU Lesser General Public License along with the GNU MP Library.  If not,
  31 dnl  see https://www.gnu.org/licenses/.
  32
  33 include(`../config.m4')
  34
  35 C            cycles/limb
  36 C AMD K8,K9     n/a
  37 C AMD K10       n/a
  38 C AMD bull      n/a
  39 C AMD pile      n/a
  40 C AMD steam      ?
  41 C AMD bobcat    n/a
  42 C AMD jaguar     ?
  43 C Intel P4      n/a
  44 C Intel core    n/a
  45 C Intel NHM     n/a
  46 C Intel SBR     n/a
  47 C Intel IBR     n/a
  48 C Intel HWL      2.32
  49 C Intel BWL      ?
  50 C Intel atom    n/a
  51 C VIA nano      n/a
  52
  53 C The loop of this code is the result of running a code generation and
  54 C optimisation tool suite written by David Harvey and Torbjörn Granlund.
  55
  56 C TODO
  57 C  * Handle small n separately, for lower overhead.
  58
  59 define(`rp',      `%rdi')   C rcx
  60 define(`up',      `%rsi')   C rdx
  61 define(`n_param', `%rdx')   C r8
  62 define(`v0_param',`%rcx')   C r9
  63
  64 define(`n',       `%rbp')
  65 define(`v0',      `%rdx')
  66
  67 ifdef(`OPERATION_addmul_1',`
  68   define(`ADDSUB',        `add')
  69   define(`ADCSBB',        `adc')
  70   define(`func',  `mpn_addmul_1')
  71 ')
  72 ifdef(`OPERATION_submul_1',`
  73   define(`ADDSUB',        `sub')
  74   define(`ADCSBB',        `sbb')
  75   define(`func',  `mpn_submul_1')
  76 ')
  77
  78 ABI_SUPPORT(DOS64)
  79 ABI_SUPPORT(STD64)
  80
  81 MULFUNC_PROLOGUE(mpn_addmul_1 mpn_submul_1)
  82
  83 ASM_START()
  84         TEXT
  85         ALIGN(16)
  86 PROLOGUE(func)
  87         FUNC_ENTRY(4)
  88         push    %rbx
  89         push    %rbp
  90         push    %r12
  91         push    %r13
  92
  93         mov     n_param, n
  94         mov     v0_param, v0
  95
  96         test    $1, R8(n)
  97         jnz     L(bx1)
  98
  99 L(bx0): shr     $2, n
 100         jc      L(b10)
 101
 102 L(b00): mulx(   (up), %r13, %r12)
 103         mulx(   8,(up), %rbx, %rax)
 104         add     %r12, %rbx
 105         adc     $0, %rax
 106         mov     (rp), %r12
 107         mov     8(rp), %rcx
 108         mulx(   16,(up), %r9, %r8)
 109         lea     -16(rp), rp
 110         lea     16(up), up
 111         ADDSUB  %r13, %r12
 112         jmp     L(lo0)
 113
 114 L(bx1): shr     $2, n
 115         jc      L(b11)
 116
 117 L(b01): mulx(   (up), %r11, %r10)
 118         jnz     L(gt1)
 119 L(n1):  ADDSUB  %r11, (rp)
 120         mov     $0, R32(%rax)
 121         adc     %r10, %rax
 122         jmp     L(ret)
 123
 124 L(gt1): mulx(   8,(up), %r13, %r12)
 125         mulx(   16,(up), %rbx, %rax)
 126         lea     24(up), up
 127         add     %r10, %r13
 128         adc     %r12, %rbx
 129         adc     $0, %rax
 130         mov     (rp), %r10
 131         mov     8(rp), %r12
 132         mov     16(rp), %rcx
 133         lea     -8(rp), rp
 134         ADDSUB  %r11, %r10
 135         jmp     L(lo1)
 136
 137 L(b11): mulx(   (up), %rbx, %rax)
 138         mov     (rp), %rcx
 139         mulx(   8,(up), %r9, %r8)
 140         lea     8(up), up
 141         lea     -24(rp), rp
 142         inc     n                       C adjust n
 143         ADDSUB  %rbx, %rcx
 144         jmp     L(lo3)
 145
 146 L(b10): mulx(   (up), %r9, %r8)
 147         mulx(   8,(up), %r11, %r10)
 148         lea     -32(rp), rp
 149         mov     $0, R32(%rax)
 150         clc                             C clear cf
 151         jz      L(end)                  C depends on old shift
 152
 153         ALIGN(16)
 154 L(top): adc     %rax, %r9
 155         lea     32(rp), rp
 156         adc     %r8, %r11
 157         mulx(   16,(up), %r13, %r12)
 158         mov     (rp), %r8
 159         mulx(   24,(up), %rbx, %rax)
 160         lea     32(up), up
 161         adc     %r10, %r13
 162         adc     %r12, %rbx
 163         adc     $0, %rax
 164         mov     8(rp), %r10
 165         mov     16(rp), %r12
 166         ADDSUB  %r9, %r8
 167         mov     24(rp), %rcx
 168         mov     %r8, (rp)
 169         ADCSBB  %r11, %r10
 170 L(lo1): mulx(   (up), %r9, %r8)
 171         mov     %r10, 8(rp)
 172         ADCSBB  %r13, %r12
 173 L(lo0): mov     %r12, 16(rp)
 174         ADCSBB  %rbx, %rcx
 175 L(lo3): mulx(   8,(up), %r11, %r10)
 176         mov     %rcx, 24(rp)
 177         dec     n
 178         jnz     L(top)
 179
 180 L(end): adc     %rax, %r9
 181         adc     %r8, %r11
 182         mov     32(rp), %r8
 183         mov     %r10, %rax
 184         adc     $0, %rax
 185         mov     40(rp), %r10
 186         ADDSUB  %r9, %r8
 187         mov     %r8, 32(rp)
 188         ADCSBB  %r11, %r10
 189         mov     %r10, 40(rp)
 190         adc     $0, %rax
 191
 192 L(ret): pop     %r13
 193         pop     %r12
 194         pop     %rbp
 195         pop     %rbx
 196         FUNC_EXIT()
 197         ret
 198 EPILOGUE()