arch/x86_64-all/exec/copymem.c

   1 /*
   2     Copyright © 2009-2012, The AROS Development Team. All rights reserved.
   3     $Id$
   4
   5     Desc: Copy memory.
   6     Lang: english
   7 */
   8
   9 #define DEBUG 0
  10 #include <aros/debug.h>
  11
  12 #include <aros/libcall.h>
  13 #include <proto/exec.h>
  14
  15 #define SSE_REG_SIZE 16
  16 #define SSE_REG_MASK 0xF
  17
  18 #define MEMFENCE __asm__ __volatile__ ("sfence":::"memory")
  19 #define MMENABLE __asm__ __volatile__ ("emms":::"memory")
  20
  21 #define __byte_memcpy(src,dst,size)                             \
  22 {                                                               \
  23     __asm__ __volatile__(                                       \
  24         "    rep; movsb"                                        \
  25         : "=&D" (dst), "=&S" (src), "=&c" (dummy)               \
  26         : "0" (dst), "1" (src), "2" (size)                      \
  27         : "memory");                                            \
  28 }
  29
  30 #define __long_memcpy(src,dst,size)                             \
  31 {                                                               \
  32     __asm__ __volatile__(                                       \
  33         "    rep; movsl" "\n\t"                                 \
  34         "    testb $2,%b6" "\n\t"                               \
  35         "    je 1f" "\n\t"                                      \
  36         "    movsw" "\n"                                        \
  37         "1:  testb $1,%b6" "\n\t"                               \
  38         "    je 2f" "\n\t"                                      \
  39         "    movsb" "\n"                                        \
  40         "2:"                                                    \
  41         : "=&D" (dst), "=&S" (src), "=&c" (dummy)               \
  42         : "0" (dst), "1" (src), "2" (size >> 2), "q" (size)     \
  43         : "memory");                                            \
  44 }
  45
  46 static __inline__ void __small_memcpy(const void * src, void * dst, ULONG size)
  47 {
  48     register unsigned long int dummy;
  49     if( size < 4 ) {
  50 D(bug("[Exec] __byte_memcpy(%p, %p, %ld)\n", src, dst, size));
  51
  52         __byte_memcpy(src, dst, size);
  53     }
  54     else
  55     {
  56 D(bug("[Exec] __long_memcpy(%p, %p, %ld)\n", src, dst, size));
  57
  58         __long_memcpy(src, dst, size);
  59     }
  60 }
  61
  62 /*****************************************************************************
  63
  64     NAME */
  65
  66         AROS_LH3I(void, CopyMem,
  67
  68 /*  SYNOPSIS */
  69         AROS_LHA(CONST_APTR, source, A0),
  70         AROS_LHA(APTR, dest, A1),
  71         AROS_LHA(IPTR, size, D0),
  72
  73 /*  LOCATION */
  74         struct ExecBase *, SysBase, 104, Exec)
  75
  76 /*  FUNCTION
  77         Copy some data from one location to another in memory using
  78         SSE optimised copying method if enough data.
  79
  80     INPUTS
  81         source - Pointer to source area
  82         dest   - Pointer to destination
  83         size   - number of bytes to copy
  84
  85     RESULT
  86
  87     NOTES
  88         The source and destination area are not allowed to overlap.
  89         If the src isn't on a 16-byte boundary, it is aligned
  90         first (so long as there's enough data)
  91         Copies using 4x16-byte registers = 64 bytes at a time.
  92
  93     EXAMPLE
  94
  95     BUGS
  96
  97     SEE ALSO
  98         CopyMemQuick()
  99
 100     INTERNALS
 101         64-bit sizes are not handled yet.
 102
 103 ******************************************************************************/
 104 {
 105     AROS_LIBFUNC_INIT
 106
 107     if (!size) return;
 108
 109     ULONG lcnt = (size >> 6);                   /* size / 64       */
 110
 111     const void *src = source;
 112     void *dst = dest;
 113
 114 D(bug("[Exec] CopyMem(%p, %p, %ld)\n", src, dst, size));
 115
 116     __asm__ __volatile__ (
 117                 "    prefetchnta (%0)\n"
 118                 "    prefetchnta 32(%0)\n"
 119                 "    prefetchnta 64(%0)\n"
 120                 "    prefetchnta 96(%0)\n"
 121                 "    prefetchnta 128(%0)\n"
 122                 "    prefetchnta 160(%0)\n"
 123                 "    prefetchnta 192(%0)\n"
 124                 "    prefetchnta 256(%0)\n"
 125                 "    prefetchnta 288(%0)\n"
 126                 :
 127                 : "r" (src) );
 128
 129     if ((lcnt > 0) && (size >= (SSE_REG_SIZE * 4)))
 130     {
 131 D(bug("[Exec] CopyMem: Using SSE Copy.\n"));
 132         ULONG alignsize = ((SSE_REG_SIZE - ((IPTR)src & SSE_REG_MASK)));
 133
 134         if ((((IPTR)src & SSE_REG_MASK) != 0) && (((IPTR)(dst + alignsize) & SSE_REG_MASK) == 0))
 135         {
 136 D(bug("[Exec] CopyMem: Aligning src to %d byte boundary (%d bytes) .. \n", SSE_REG_SIZE, alignsize));
 137
 138             __small_memcpy(src, dst, alignsize);
 139
 140             size -= alignsize;
 141             lcnt = (size >> 6);                 /* size / 64       */
 142             src += alignsize;
 143             dst += alignsize;
 144         }
 145         if (lcnt > 0) {
 146             if ((((IPTR)src & SSE_REG_MASK) == 0) && (((IPTR)dst & SSE_REG_MASK) == 0))
 147             {
 148                 /*
 149                     # SRC and DST aligned on 16-byte boundary.
 150                     We can use movaps instead of movups since we meet
 151                     the alignment constraints (a general-protection fault
 152                     would be triggered otherwise)
 153                 */
 154                 size -= (lcnt << 6);
 155                 for( ; lcnt > 0; lcnt--)
 156                 {
 157 D(bug("[Exec] CopyMem: SSE Aligned-Copy %p to %p.\n", src, dst));
 158
 159                     __asm__ __volatile__ (
 160                         "    prefetchnta 320(%0)\n"
 161                         "    prefetchnta 352(%0)\n"
 162                         "    movaps (%0), %%xmm0\n"
 163                         "    movaps 16(%0), %%xmm1\n"
 164                         "    movaps 32(%0), %%xmm2\n"
 165                         "    movaps 48(%0), %%xmm3\n"
 166                         "    movntps %%xmm0, (%1)\n"
 167                         "    movntps %%xmm1, 16(%1)\n"
 168                         "    movntps %%xmm2, 32(%1)\n"
 169                         "    movntps %%xmm3, 48(%1)\n"
 170                         :
 171                         : "r" (src), "r" (dst)
 172                         : "memory");
 173
 174                     src += (SSE_REG_SIZE * 4);
 175                     dst += (SSE_REG_SIZE * 4);
 176                 }
 177             }
 178             else if (((IPTR)dst & SSE_REG_MASK) == 0)
 179             {
 180                 /*
 181                     # SRC is unaligned and DST aligned on 16-byte boundary.
 182                 */
 183                 size -= (lcnt << 6);
 184                 for( ; lcnt > 0; lcnt--)
 185                 {
 186 D(bug("[Exec] CopyMem: SSE Unaligned-Copy %p to %p.\n", src, dst));
 187
 188                     __asm__ __volatile__ (
 189                         "    prefetchnta 320(%0)\n"
 190                         "    prefetchnta 352(%0)\n"
 191                         "    movups (%0), %%xmm0\n"
 192                         "    movups 16(%0), %%xmm1\n"
 193                         "    movups 32(%0), %%xmm2\n"
 194                         "    movups 48(%0), %%xmm3\n"
 195                         "    movntps %%xmm0, (%1)\n"
 196                         "    movntps %%xmm1, 16(%1)\n"
 197                         "    movntps %%xmm2, 32(%1)\n"
 198                         "    movntps %%xmm3, 48(%1)\n"
 199                         :
 200                         : "r" (src), "r" (dst)
 201                         : "memory");
 202
 203                     src += (SSE_REG_SIZE * 4);
 204                     dst += (SSE_REG_SIZE * 4);
 205                 }
 206             }
 207         }
 208     }
 209
 210     if (size > 0)
 211     {
 212 D(bug("[Exec] CopyMem: Copy remaining %ld bytes.\n", size));
 213         __small_memcpy(src, dst, size);
 214     }
 215
 216     /*
 217         FENCE Memory to re-order again since movntq is weakly-ordered ?
 218     */
 219     MEMFENCE;
 220     /*
 221         enable FPU use ?
 222     */
 223     MMENABLE;
 224
 225     D(bug("[Exec] CopyMem: Finished.\n"));
 226
 227     AROS_LIBFUNC_EXIT
 228 } /* CopyMem */
 229