arch/x86_64-all/exec/copymem.c

   1 /*
   2     Copyright © 2009, The AROS Development Team. All rights reserved.
   3     $Id$
   4
   5     Desc: Copy memory.
   6     Lang: english
   7 */
   8
   9 #define DEBUG 0
  10 #include <aros/debug.h>
  11
  12 #include <aros/libcall.h>
  13 #include <proto/exec.h>
  14
  15 #define SSE_REG_SIZE 16
  16 #define SSE_REG_MASK 0xF
  17
  18 #define MEMFENCE __asm__ __volatile__ ("sfence":::"memory")
  19 #define MMENABLE __asm__ __volatile__ ("emms":::"memory")
  20
  21 #define __byte_memcpy(src,dst,size)                             \
  22 {                                                               \
  23     __asm__ __volatile__(                                       \
  24         "    rep; movsb"                                        \
  25         : "=&D" (dst), "=&S" (src), "=&c" (dummy)               \
  26         : "0" (dst), "1" (src), "2" (size)                      \
  27         : "memory");                                            \
  28 }
  29
  30 #define __long_memcpy(src,dst,size)                             \
  31 {                                                               \
  32     __asm__ __volatile__(                                       \
  33         "    rep; movsl" "\n\t"                                 \
  34         "    testb $2,%b6" "\n\t"                               \
  35         "    je 1f" "\n\t"                                      \
  36         "    movsw" "\n"                                        \
  37         "1:  testb $1,%b6" "\n\t"                               \
  38         "    je 2f" "\n\t"                                      \
  39         "    movsb" "\n"                                        \
  40         "2:"                                                    \
  41         : "=&D" (dst), "=&S" (src), "=&c" (dummy)               \
  42         : "0" (dst), "1" (src), "2" (size >> 2), "q" (size)     \
  43         : "memory");                                            \
  44 }
  45
  46 static __inline__ void __small_memcpy(const void * src, void * dst, ULONG size)
  47 {
  48     register unsigned long int dummy;
  49     if( size < 4 ) {
  50 D(bug("[Exec] __byte_memcpy(%p, %p, %d)\n", src, dst, size));
  51
  52         __byte_memcpy(src, dst, size);
  53     }
  54     else
  55     {
  56 D(bug("[Exec] __long_memcpy(%p, %p, %d)\n", src, dst, size));
  57
  58         __long_memcpy(src, dst, size);
  59     }
  60 }
  61
  62 /*****************************************************************************
  63
  64     NAME */
  65
  66         AROS_LH3I(void, CopyMem,
  67
  68 /*  SYNOPSIS */
  69         AROS_LHA(CONST_APTR,  source, A0),
  70         AROS_LHA(APTR,  dest,   A1),
  71         AROS_LHA(ULONG, size,   D0),
  72
  73 /*  LOCATION */
  74         struct ExecBase *, SysBase, 104, Exec)
  75
  76 /*  FUNCTION
  77         Copy some data from one location to another in memory using
  78         SSE optimised copying method if enough data.
  79
  80     INPUTS
  81         source - Pointer to source area
  82         dest   - Pointer to destination
  83         size   - number of bytes to copy
  84
  85     RESULT
  86
  87     NOTES
  88         The source and destination area are not allowed to overlap.
  89         If the src isnt on a 16byte boundary it is aligned
  90         first (so long as theres enough data..)
  91         Copies using 4x16byte registers = 64bytes at a time.
  92
  93     EXAMPLE
  94
  95     BUGS
  96
  97     SEE ALSO
  98         CopyMemQuick()
  99
 100     INTERNALS
 101
 102 ******************************************************************************/
 103 {
 104     AROS_LIBFUNC_INIT
 105
 106     if (!size) return;
 107
 108     ULONG lcnt = (size >> 6);                   /* size / 64       */
 109
 110     const void *src = source;
 111     void *dst = dest;
 112
 113 D(bug("[Exec] CopyMem(%p, %p, %d)\n", src, dst, size));
 114
 115     __asm__ __volatile__ (
 116                 "    prefetchnta (%0)\n"
 117                 "    prefetchnta 32(%0)\n"
 118                 "    prefetchnta 64(%0)\n"
 119                 "    prefetchnta 96(%0)\n"
 120                 "    prefetchnta 128(%0)\n"
 121                 "    prefetchnta 160(%0)\n"
 122                 "    prefetchnta 192(%0)\n"
 123                 "    prefetchnta 256(%0)\n"
 124                 "    prefetchnta 288(%0)\n"
 125                 :
 126                 : "r" (src) );
 127
 128     if ((lcnt > 0) && (size >= (SSE_REG_SIZE * 4)))
 129     {
 130 D(bug("[Exec] CopyMem: Using SSE Copy.\n"));
 131         ULONG alignsize = ((SSE_REG_SIZE - ((IPTR)src & SSE_REG_MASK)));
 132
 133         if ((((IPTR)src & SSE_REG_MASK) != 0) && (((IPTR)(dst + alignsize) & SSE_REG_MASK) == 0))
 134         {
 135 D(bug("[Exec] CopyMem: Aligning src to %d byte boundary (%d bytes) .. \n", SSE_REG_SIZE, alignsize));
 136
 137             __small_memcpy(src, dst, alignsize);
 138
 139             size -= alignsize;
 140             lcnt = (size >> 6);                 /* size / 64       */
 141             src += alignsize;
 142             dst += alignsize;
 143         }
 144         if (lcnt > 0) {
 145             if ((((IPTR)src & SSE_REG_MASK) == 0) && (((IPTR)dst & SSE_REG_MASK) == 0))
 146             {
 147                 /*
 148                     # SRC and DST aligned on 16-byte boundary.
 149                     We can use movaps instead of movups since we meet
 150                     the alignment constraints (a general-protection fault
 151                     would be triggered otherwise)
 152                 */
 153                 size -= (lcnt << 6);
 154                 for( ; lcnt > 0; lcnt--)
 155                 {
 156 D(bug("[Exec] CopyMem: SSE Aligned-Copy %p to %p.\n", src, dst));
 157
 158                     __asm__ __volatile__ (
 159                         "    prefetchnta 320(%0)\n"
 160                         "    prefetchnta 352(%0)\n"
 161                         "    movaps (%0), %%xmm0\n"
 162                         "    movaps 16(%0), %%xmm1\n"
 163                         "    movaps 32(%0), %%xmm2\n"
 164                         "    movaps 48(%0), %%xmm3\n"
 165                         "    movntps %%xmm0, (%1)\n"
 166                         "    movntps %%xmm1, 16(%1)\n"
 167                         "    movntps %%xmm2, 32(%1)\n"
 168                         "    movntps %%xmm3, 48(%1)\n"
 169                         :
 170                         : "r" (src), "r" (dst)
 171                         : "memory");
 172
 173                     src += (SSE_REG_SIZE * 4);
 174                     dst += (SSE_REG_SIZE * 4);
 175                 }
 176             }
 177             else if (((IPTR)dst & SSE_REG_MASK) == 0)
 178             {
 179                 /*
 180                     # SRC is unaligned and DST aligned on 16-byte boundary.
 181                 */
 182                 size -= (lcnt << 6);
 183                 for( ; lcnt > 0; lcnt--)
 184                 {
 185 D(bug("[Exec] CopyMem: SSE Unaligned-Copy %p to %p.\n", src, dst));
 186
 187                     __asm__ __volatile__ (
 188                         "    prefetchnta 320(%0)\n"
 189                         "    prefetchnta 352(%0)\n"
 190                         "    movups (%0), %%xmm0\n"
 191                         "    movups 16(%0), %%xmm1\n"
 192                         "    movups 32(%0), %%xmm2\n"
 193                         "    movups 48(%0), %%xmm3\n"
 194                         "    movntps %%xmm0, (%1)\n"
 195                         "    movntps %%xmm1, 16(%1)\n"
 196                         "    movntps %%xmm2, 32(%1)\n"
 197                         "    movntps %%xmm3, 48(%1)\n"
 198                         :
 199                         : "r" (src), "r" (dst)
 200                         : "memory");
 201
 202                     src += (SSE_REG_SIZE * 4);
 203                     dst += (SSE_REG_SIZE * 4);
 204                 }
 205             }
 206         }
 207     }
 208
 209     if (size > 0)
 210     {
 211 D(bug("[Exec] CopyMem: Copy remaining %d bytes.\n", size));
 212         __small_memcpy(src, dst, size);
 213     }
 214
 215     /*
 216         FENCE Memory to re-order again since movntq is weakly-ordered ?
 217     */
 218     MEMFENCE;
 219     /*
 220         enable FPU use ?
 221     */
 222     MMENABLE;
 223
 224     D(bug("[Exec] CopyMem: Finished.\n"));
 225
 226     AROS_LIBFUNC_EXIT
 227 } /* CopyMem */
 228