| blob | 749c82d3797c837974b764576613ba829c10fd39 |
1 /* memcpy with SSSE3
2 Copyright (C) 2010 Free Software Foundation, Inc.
3 Contributed by Intel Corporation.
4 This file is part of the GNU C Library.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, write to the Free
18 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
19 02111-1307 USA. */
21 #include <sysdep.h>
23 #if !defined NOT_IN_libc \
24 && (defined SHARED \
25 || defined USE_AS_MEMMOVE \
26 || !defined USE_MULTIARCH)
28 #include "asm-syntax.h"
30 #ifndef MEMCPY
31 # define MEMCPY __memcpy_ssse3
32 # define MEMCPY_CHK __memcpy_chk_ssse3
33 #endif
35 #ifdef USE_AS_BCOPY
36 # define SRC PARMS
37 # define DEST SRC+4
38 # define LEN DEST+4
39 #else
40 # define DEST PARMS
41 # define SRC DEST+4
42 # define LEN SRC+4
43 #endif
45 #define CFI_PUSH(REG) \
46 cfi_adjust_cfa_offset (4); \
47 cfi_rel_offset (REG, 0)
49 #define CFI_POP(REG) \
50 cfi_adjust_cfa_offset (-4); \
51 cfi_restore (REG)
53 #define PUSH(REG) pushl REG; CFI_PUSH (REG)
54 #define POP(REG) popl REG; CFI_POP (REG)
56 #ifdef SHARED
57 # define PARMS 8 /* Preserve EBX. */
58 # define ENTRANCE PUSH (%ebx);
59 # define RETURN_END POP (%ebx); ret
60 # define RETURN RETURN_END; CFI_PUSH (%ebx)
61 # define JMPTBL(I, B) I - B
63 /* Load an entry in a jump table into EBX and branch to it. TABLE is a
64 jump table with relative offsets. INDEX is a register contains the
65 index into the jump table. SCALE is the scale of INDEX. */
66 # define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \
67 /* We first load PC into EBX. */ \
68 call __i686.get_pc_thunk.bx; \
69 /* Get the address of the jump table. */ \
70 addl $(TABLE - .), %ebx; \
71 /* Get the entry and convert the relative offset to the \
72 absolute address. */ \
73 addl (%ebx,INDEX,SCALE), %ebx; \
74 /* We loaded the jump table. Go. */ \
75 jmp *%ebx
77 # define BRANCH_TO_JMPTBL_ENTRY_VALUE(TABLE) \
78 addl $(TABLE - .), %ebx
80 # define BRANCH_TO_JMPTBL_ENTRY_TAIL(TABLE, INDEX, SCALE) \
81 addl (%ebx,INDEX,SCALE), %ebx; \
82 /* We loaded the jump table. Go. */ \
83 jmp *%ebx
85 .section .gnu.linkonce.t.__i686.get_pc_thunk.bx,"ax",@progbits
86 .globl __i686.get_pc_thunk.bx
87 .hidden __i686.get_pc_thunk.bx
88 ALIGN (4)
89 .type __i686.get_pc_thunk.bx,@function
90 __i686.get_pc_thunk.bx:
91 movl (%esp), %ebx
92 ret
93 #else
94 # define PARMS 4
95 # define ENTRANCE
96 # define RETURN_END ret
97 # define RETURN RETURN_END
98 # define JMPTBL(I, B) I
100 /* Branch to an entry in a jump table. TABLE is a jump table with
101 absolute offsets. INDEX is a register contains the index into the
102 jump table. SCALE is the scale of INDEX. */
103 # define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \
104 jmp *TABLE(,INDEX,SCALE)
106 # define BRANCH_TO_JMPTBL_ENTRY_VALUE(TABLE)
108 # define BRANCH_TO_JMPTBL_ENTRY_TAIL(TABLE, INDEX, SCALE) \
109 jmp *TABLE(,INDEX,SCALE)
110 #endif
112 .section .text.ssse3,"ax",@progbits
113 #if defined SHARED && !defined NOT_IN_libc && !defined USE_AS_BCOPY
114 ENTRY (MEMCPY_CHK)
115 movl 12(%esp), %eax
116 cmpl %eax, 16(%esp)
117 jb HIDDEN_JUMPTARGET (__chk_fail)
118 END (MEMCPY_CHK)
119 #endif
120 ENTRY (MEMCPY)
121 ENTRANCE
122 movl LEN(%esp), %ecx
123 movl SRC(%esp), %eax
124 movl DEST(%esp), %edx
126 #ifdef USE_AS_MEMMOVE
127 cmp %eax, %edx
128 jb L(copy_forward)
129 je L(fwd_write_0bytes)
130 cmp $32, %ecx
131 jge L(memmove_bwd)
132 jmp L(bk_write_less32bytes_2)
133 L(memmove_bwd):
134 add %ecx, %eax
135 cmp %eax, %edx
136 movl SRC(%esp), %eax
137 jb L(copy_backward)
139 L(copy_forward):
140 #endif
141 cmp $48, %ecx
142 jge L(48bytesormore)
144 L(fwd_write_less32bytes):
145 #ifndef USE_AS_MEMMOVE
146 cmp %dl, %al
147 jl L(bk_write)
148 #endif
149 add %ecx, %edx
150 add %ecx, %eax
151 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
152 #ifndef USE_AS_MEMMOVE
153 L(bk_write):
154 BRANCH_TO_JMPTBL_ENTRY (L(table_48_bytes_bwd), %ecx, 4)
155 #endif
157 ALIGN (4)
158 /* ECX > 32 and EDX is 4 byte aligned. */
159 L(48bytesormore):
160 movdqu (%eax), %xmm0
161 PUSH (%edi)
162 movl %edx, %edi
163 and $-16, %edx
164 PUSH (%esi)
165 add $16, %edx
166 movl %edi, %esi
167 sub %edx, %edi
168 add %edi, %ecx
169 sub %edi, %eax
171 #ifdef SHARED_CACHE_SIZE_HALF
172 cmp $SHARED_CACHE_SIZE_HALF, %ecx
173 #else
174 # ifdef SHARED
175 call __i686.get_pc_thunk.bx
176 add $_GLOBAL_OFFSET_TABLE_, %ebx
177 cmp __x86_shared_cache_size_half@GOTOFF(%ebx), %ecx
178 # else
179 cmp __x86_shared_cache_size_half, %ecx
180 # endif
181 #endif
183 mov %eax, %edi
184 jge L(large_page)
185 and $0xf, %edi
186 jz L(shl_0)
188 BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %edi, 4)
190 ALIGN (4)
191 L(shl_0):
192 movdqu %xmm0, (%esi)
193 xor %edi, %edi
194 POP (%esi)
195 cmp $127, %ecx
196 ja L(shl_0_gobble)
197 lea -32(%ecx), %ecx
198 L(shl_0_loop):
199 movdqa (%eax, %edi), %xmm0
200 movdqa 16(%eax, %edi), %xmm1
201 sub $32, %ecx
202 movdqa %xmm0, (%edx, %edi)
203 movdqa %xmm1, 16(%edx, %edi)
204 lea 32(%edi), %edi
205 jl L(shl_0_end)
207 movdqa (%eax, %edi), %xmm0
208 movdqa 16(%eax, %edi), %xmm1
209 sub $32, %ecx
210 movdqa %xmm0, (%edx, %edi)
211 movdqa %xmm1, 16(%edx, %edi)
212 lea 32(%edi), %edi
213 jl L(shl_0_end)
215 movdqa (%eax, %edi), %xmm0
216 movdqa 16(%eax, %edi), %xmm1
217 sub $32, %ecx
218 movdqa %xmm0, (%edx, %edi)
219 movdqa %xmm1, 16(%edx, %edi)
220 lea 32(%edi), %edi
221 jl L(shl_0_end)
223 movdqa (%eax, %edi), %xmm0
224 movdqa 16(%eax, %edi), %xmm1
225 sub $32, %ecx
226 movdqa %xmm0, (%edx, %edi)
227 movdqa %xmm1, 16(%edx, %edi)
228 lea 32(%edi), %edi
229 L(shl_0_end):
230 lea 32(%ecx), %ecx
231 add %ecx, %edi
232 add %edi, %edx
233 add %edi, %eax
234 POP (%edi)
235 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
237 L(shl_0_gobble):
239 #ifdef DATA_CACHE_SIZE_HALF
240 cmp $DATA_CACHE_SIZE_HALF, %ecx
241 #else
242 # ifdef SHARED
243 call __i686.get_pc_thunk.bx
244 add $_GLOBAL_OFFSET_TABLE_, %ebx
245 cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
246 # else
247 cmp __x86_data_cache_size_half, %ecx
248 # endif
249 #endif
251 POP (%edi)
252 lea -128(%ecx), %ecx
253 jge L(shl_0_gobble_mem_loop)
254 L(shl_0_gobble_cache_loop):
255 movdqa (%eax), %xmm0
256 movdqa 0x10(%eax), %xmm1
257 movdqa 0x20(%eax), %xmm2
258 movdqa 0x30(%eax), %xmm3
259 movdqa 0x40(%eax), %xmm4
260 movdqa 0x50(%eax), %xmm5
261 movdqa 0x60(%eax), %xmm6
262 movdqa 0x70(%eax), %xmm7
263 lea 0x80(%eax), %eax
264 sub $128, %ecx
265 movdqa %xmm0, (%edx)
266 movdqa %xmm1, 0x10(%edx)
267 movdqa %xmm2, 0x20(%edx)
268 movdqa %xmm3, 0x30(%edx)
269 movdqa %xmm4, 0x40(%edx)
270 movdqa %xmm5, 0x50(%edx)
271 movdqa %xmm6, 0x60(%edx)
272 movdqa %xmm7, 0x70(%edx)
273 lea 0x80(%edx), %edx
275 jge L(shl_0_gobble_cache_loop)
276 L(shl_0_gobble_cache_loop_tail):
277 cmp $-0x40, %ecx
278 lea 0x80(%ecx), %ecx
279 jl L(shl_0_cache_less_64bytes)
281 movdqa (%eax), %xmm0
282 sub $0x40, %ecx
283 movdqa 0x10(%eax), %xmm1
285 movdqa %xmm0, (%edx)
286 movdqa %xmm1, 0x10(%edx)
288 movdqa 0x20(%eax), %xmm0
289 movdqa 0x30(%eax), %xmm1
290 add $0x40, %eax
292 movdqa %xmm0, 0x20(%edx)
293 movdqa %xmm1, 0x30(%edx)
294 add $0x40, %edx
295 L(shl_0_cache_less_64bytes):
296 cmp $0x20, %ecx
297 jl L(shl_0_cache_less_32bytes)
298 movdqa (%eax), %xmm0
299 sub $0x20, %ecx
300 movdqa 0x10(%eax), %xmm1
301 add $0x20, %eax
302 movdqa %xmm0, (%edx)
303 movdqa %xmm1, 0x10(%edx)
304 add $0x20, %edx
305 L(shl_0_cache_less_32bytes):
306 cmp $0x10, %ecx
307 jl L(shl_0_cache_less_16bytes)
308 sub $0x10, %ecx
309 movdqa (%eax), %xmm0
310 add $0x10, %eax
311 movdqa %xmm0, (%edx)
312 add $0x10, %edx
313 L(shl_0_cache_less_16bytes):
314 add %ecx, %edx
315 add %ecx, %eax
316 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
319 ALIGN (4)
320 L(shl_0_gobble_mem_loop):
321 prefetcht0 0x1c0(%eax)
322 prefetcht0 0x280(%eax)
323 prefetcht0 0x1c0(%edx)
325 movdqa (%eax), %xmm0
326 movdqa 0x10(%eax), %xmm1
327 movdqa 0x20(%eax), %xmm2
328 movdqa 0x30(%eax), %xmm3
329 movdqa 0x40(%eax), %xmm4
330 movdqa 0x50(%eax), %xmm5
331 movdqa 0x60(%eax), %xmm6
332 movdqa 0x70(%eax), %xmm7
333 lea 0x80(%eax), %eax
334 sub $0x80, %ecx
335 movdqa %xmm0, (%edx)
336 movdqa %xmm1, 0x10(%edx)
337 movdqa %xmm2, 0x20(%edx)
338 movdqa %xmm3, 0x30(%edx)
339 movdqa %xmm4, 0x40(%edx)
340 movdqa %xmm5, 0x50(%edx)
341 movdqa %xmm6, 0x60(%edx)
342 movdqa %xmm7, 0x70(%edx)
343 lea 0x80(%edx), %edx
345 jge L(shl_0_gobble_mem_loop)
346 cmp $-0x40, %ecx
347 lea 0x80(%ecx), %ecx
348 jl L(shl_0_mem_less_64bytes)
350 movdqa (%eax), %xmm0
351 sub $0x40, %ecx
352 movdqa 0x10(%eax), %xmm1
354 movdqa %xmm0, (%edx)
355 movdqa %xmm1, 0x10(%edx)
357 movdqa 0x20(%eax), %xmm0
358 movdqa 0x30(%eax), %xmm1
359 add $0x40, %eax
361 movdqa %xmm0, 0x20(%edx)
362 movdqa %xmm1, 0x30(%edx)
363 add $0x40, %edx
364 L(shl_0_mem_less_64bytes):
365 cmp $0x20, %ecx
366 jl L(shl_0_mem_less_32bytes)
367 movdqa (%eax), %xmm0
368 sub $0x20, %ecx
369 movdqa 0x10(%eax), %xmm1
370 add $0x20, %eax
371 movdqa %xmm0, (%edx)
372 movdqa %xmm1, 0x10(%edx)
373 add $0x20, %edx
374 L(shl_0_mem_less_32bytes):
375 cmp $0x10, %ecx
376 jl L(shl_0_mem_less_16bytes)
377 sub $0x10, %ecx
378 movdqa (%eax), %xmm0
379 add $0x10, %eax
380 movdqa %xmm0, (%edx)
381 add $0x10, %edx
382 L(shl_0_mem_less_16bytes):
383 add %ecx, %edx
384 add %ecx, %eax
385 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
388 ALIGN (4)
389 L(shl_1):
390 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
391 lea -1(%eax), %eax
392 movaps (%eax), %xmm1
393 xor %edi, %edi
394 lea -32(%ecx), %ecx
395 movdqu %xmm0, (%esi)
396 POP (%esi)
397 L(shl_1_loop):
399 movdqa 16(%eax, %edi), %xmm2
400 sub $32, %ecx
401 movdqa 32(%eax, %edi), %xmm3
402 movdqa %xmm3, %xmm4
403 palignr $1, %xmm2, %xmm3
404 palignr $1, %xmm1, %xmm2
405 lea 32(%edi), %edi
406 movdqa %xmm2, -32(%edx, %edi)
407 movdqa %xmm3, -16(%edx, %edi)
409 jl L(shl_1_end)
411 movdqa 16(%eax, %edi), %xmm2
412 sub $32, %ecx
413 movdqa 32(%eax, %edi), %xmm3
414 movdqa %xmm3, %xmm1
415 palignr $1, %xmm2, %xmm3
416 palignr $1, %xmm4, %xmm2
417 lea 32(%edi), %edi
418 movdqa %xmm2, -32(%edx, %edi)
419 movdqa %xmm3, -16(%edx, %edi)
421 jae L(shl_1_loop)
423 L(shl_1_end):
424 lea 32(%ecx), %ecx
425 add %ecx, %edi
426 add %edi, %edx
427 lea 1(%edi, %eax), %eax
428 POP (%edi)
429 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
431 ALIGN (4)
432 L(shl_2):
433 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
434 lea -2(%eax), %eax
435 movaps (%eax), %xmm1
436 xor %edi, %edi
437 lea -32(%ecx), %ecx
438 movdqu %xmm0, (%esi)
439 POP (%esi)
440 L(shl_2_loop):
442 movdqa 16(%eax, %edi), %xmm2
443 sub $32, %ecx
444 movdqa 32(%eax, %edi), %xmm3
445 movdqa %xmm3, %xmm4
446 palignr $2, %xmm2, %xmm3
447 palignr $2, %xmm1, %xmm2
448 lea 32(%edi), %edi
449 movdqa %xmm2, -32(%edx, %edi)
450 movdqa %xmm3, -16(%edx, %edi)
452 jl L(shl_2_end)
454 movdqa 16(%eax, %edi), %xmm2
455 sub $32, %ecx
456 movdqa 32(%eax, %edi), %xmm3
457 movdqa %xmm3, %xmm1
458 palignr $2, %xmm2, %xmm3
459 palignr $2, %xmm4, %xmm2
460 lea 32(%edi), %edi
461 movdqa %xmm2, -32(%edx, %edi)
462 movdqa %xmm3, -16(%edx, %edi)
464 jae L(shl_2_loop)
466 L(shl_2_end):
467 lea 32(%ecx), %ecx
468 add %ecx, %edi
469 add %edi, %edx
470 lea 2(%edi, %eax), %eax
471 POP (%edi)
472 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
474 ALIGN (4)
475 L(shl_3):
476 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
477 lea -3(%eax), %eax
478 movaps (%eax), %xmm1
479 xor %edi, %edi
480 lea -32(%ecx), %ecx
481 movdqu %xmm0, (%esi)
482 POP (%esi)
483 L(shl_3_loop):
485 movdqa 16(%eax, %edi), %xmm2
486 sub $32, %ecx
487 movdqa 32(%eax, %edi), %xmm3
488 movdqa %xmm3, %xmm4
489 palignr $3, %xmm2, %xmm3
490 palignr $3, %xmm1, %xmm2
491 lea 32(%edi), %edi
492 movdqa %xmm2, -32(%edx, %edi)
493 movdqa %xmm3, -16(%edx, %edi)
495 jl L(shl_3_end)
497 movdqa 16(%eax, %edi), %xmm2
498 sub $32, %ecx
499 movdqa 32(%eax, %edi), %xmm3
500 movdqa %xmm3, %xmm1
501 palignr $3, %xmm2, %xmm3
502 palignr $3, %xmm4, %xmm2
503 lea 32(%edi), %edi
504 movdqa %xmm2, -32(%edx, %edi)
505 movdqa %xmm3, -16(%edx, %edi)
507 jae L(shl_3_loop)
509 L(shl_3_end):
510 lea 32(%ecx), %ecx
511 add %ecx, %edi
512 add %edi, %edx
513 lea 3(%edi, %eax), %eax
514 POP (%edi)
515 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
517 ALIGN (4)
518 L(shl_4):
519 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
520 lea -4(%eax), %eax
521 movaps (%eax), %xmm1
522 xor %edi, %edi
523 lea -32(%ecx), %ecx
524 movdqu %xmm0, (%esi)
525 POP (%esi)
526 L(shl_4_loop):
528 movdqa 16(%eax, %edi), %xmm2
529 sub $32, %ecx
530 movdqa 32(%eax, %edi), %xmm3
531 movdqa %xmm3, %xmm4
532 palignr $4, %xmm2, %xmm3
533 palignr $4, %xmm1, %xmm2
534 lea 32(%edi), %edi
535 movdqa %xmm2, -32(%edx, %edi)
536 movdqa %xmm3, -16(%edx, %edi)
538 jl L(shl_4_end)
540 movdqa 16(%eax, %edi), %xmm2
541 sub $32, %ecx
542 movdqa 32(%eax, %edi), %xmm3
543 movdqa %xmm3, %xmm1
544 palignr $4, %xmm2, %xmm3
545 palignr $4, %xmm4, %xmm2
546 lea 32(%edi), %edi
547 movdqa %xmm2, -32(%edx, %edi)
548 movdqa %xmm3, -16(%edx, %edi)
550 jae L(shl_4_loop)
552 L(shl_4_end):
553 lea 32(%ecx), %ecx
554 add %ecx, %edi
555 add %edi, %edx
556 lea 4(%edi, %eax), %eax
557 POP (%edi)
558 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
560 ALIGN (4)
561 L(shl_5):
562 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
563 lea -5(%eax), %eax
564 movaps (%eax), %xmm1
565 xor %edi, %edi
566 lea -32(%ecx), %ecx
567 movdqu %xmm0, (%esi)
568 POP (%esi)
569 L(shl_5_loop):
571 movdqa 16(%eax, %edi), %xmm2
572 sub $32, %ecx
573 movdqa 32(%eax, %edi), %xmm3
574 movdqa %xmm3, %xmm4
575 palignr $5, %xmm2, %xmm3
576 palignr $5, %xmm1, %xmm2
577 lea 32(%edi), %edi
578 movdqa %xmm2, -32(%edx, %edi)
579 movdqa %xmm3, -16(%edx, %edi)
581 jl L(shl_5_end)
583 movdqa 16(%eax, %edi), %xmm2
584 sub $32, %ecx
585 movdqa 32(%eax, %edi), %xmm3
586 movdqa %xmm3, %xmm1
587 palignr $5, %xmm2, %xmm3
588 palignr $5, %xmm4, %xmm2
589 lea 32(%edi), %edi
590 movdqa %xmm2, -32(%edx, %edi)
591 movdqa %xmm3, -16(%edx, %edi)
593 jae L(shl_5_loop)
595 L(shl_5_end):
596 lea 32(%ecx), %ecx
597 add %ecx, %edi
598 add %edi, %edx
599 lea 5(%edi, %eax), %eax
600 POP (%edi)
601 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
604 ALIGN (4)
605 L(shl_6):
606 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
607 lea -6(%eax), %eax
608 movaps (%eax), %xmm1
609 xor %edi, %edi
610 lea -32(%ecx), %ecx
611 movdqu %xmm0, (%esi)
612 POP (%esi)
613 L(shl_6_loop):
615 movdqa 16(%eax, %edi), %xmm2
616 sub $32, %ecx
617 movdqa 32(%eax, %edi), %xmm3
618 movdqa %xmm3, %xmm4
619 palignr $6, %xmm2, %xmm3
620 palignr $6, %xmm1, %xmm2
621 lea 32(%edi), %edi
622 movdqa %xmm2, -32(%edx, %edi)
623 movdqa %xmm3, -16(%edx, %edi)
625 jl L(shl_6_end)
627 movdqa 16(%eax, %edi), %xmm2
628 sub $32, %ecx
629 movdqa 32(%eax, %edi), %xmm3
630 movdqa %xmm3, %xmm1
631 palignr $6, %xmm2, %xmm3
632 palignr $6, %xmm4, %xmm2
633 lea 32(%edi), %edi
634 movdqa %xmm2, -32(%edx, %edi)
635 movdqa %xmm3, -16(%edx, %edi)
637 jae L(shl_6_loop)
639 L(shl_6_end):
640 lea 32(%ecx), %ecx
641 add %ecx, %edi
642 add %edi, %edx
643 lea 6(%edi, %eax), %eax
644 POP (%edi)
645 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
647 ALIGN (4)
648 L(shl_7):
649 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
650 lea -7(%eax), %eax
651 movaps (%eax), %xmm1
652 xor %edi, %edi
653 lea -32(%ecx), %ecx
654 movdqu %xmm0, (%esi)
655 POP (%esi)
656 L(shl_7_loop):
658 movdqa 16(%eax, %edi), %xmm2
659 sub $32, %ecx
660 movdqa 32(%eax, %edi), %xmm3
661 movdqa %xmm3, %xmm4
662 palignr $7, %xmm2, %xmm3
663 palignr $7, %xmm1, %xmm2
664 lea 32(%edi), %edi
665 movdqa %xmm2, -32(%edx, %edi)
666 movdqa %xmm3, -16(%edx, %edi)
668 jl L(shl_7_end)
670 movdqa 16(%eax, %edi), %xmm2
671 sub $32, %ecx
672 movdqa 32(%eax, %edi), %xmm3
673 movdqa %xmm3, %xmm1
674 palignr $7, %xmm2, %xmm3
675 palignr $7, %xmm4, %xmm2
676 lea 32(%edi), %edi
677 movdqa %xmm2, -32(%edx, %edi)
678 movdqa %xmm3, -16(%edx, %edi)
680 jae L(shl_7_loop)
682 L(shl_7_end):
683 lea 32(%ecx), %ecx
684 add %ecx, %edi
685 add %edi, %edx
686 lea 7(%edi, %eax), %eax
687 POP (%edi)
688 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
690 ALIGN (4)
691 L(shl_8):
692 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
693 lea -8(%eax), %eax
694 movaps (%eax), %xmm1
695 xor %edi, %edi
696 lea -32(%ecx), %ecx
697 movdqu %xmm0, (%esi)
698 POP (%esi)
699 L(shl_8_loop):
701 movdqa 16(%eax, %edi), %xmm2
702 sub $32, %ecx
703 movdqa 32(%eax, %edi), %xmm3
704 movdqa %xmm3, %xmm4
705 palignr $8, %xmm2, %xmm3
706 palignr $8, %xmm1, %xmm2
707 lea 32(%edi), %edi
708 movdqa %xmm2, -32(%edx, %edi)
709 movdqa %xmm3, -16(%edx, %edi)
711 jl L(shl_8_end)
713 movdqa 16(%eax, %edi), %xmm2
714 sub $32, %ecx
715 movdqa 32(%eax, %edi), %xmm3
716 movdqa %xmm3, %xmm1
717 palignr $8, %xmm2, %xmm3
718 palignr $8, %xmm4, %xmm2
719 lea 32(%edi), %edi
720 movdqa %xmm2, -32(%edx, %edi)
721 movdqa %xmm3, -16(%edx, %edi)
723 jae L(shl_8_loop)
725 L(shl_8_end):
726 lea 32(%ecx), %ecx
727 add %ecx, %edi
728 add %edi, %edx
729 lea 8(%edi, %eax), %eax
730 POP (%edi)
731 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
733 ALIGN (4)
734 L(shl_9):
735 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
736 lea -9(%eax), %eax
737 movaps (%eax), %xmm1
738 xor %edi, %edi
739 lea -32(%ecx), %ecx
740 movdqu %xmm0, (%esi)
741 POP (%esi)
742 L(shl_9_loop):
744 movdqa 16(%eax, %edi), %xmm2
745 sub $32, %ecx
746 movdqa 32(%eax, %edi), %xmm3
747 movdqa %xmm3, %xmm4
748 palignr $9, %xmm2, %xmm3
749 palignr $9, %xmm1, %xmm2
750 lea 32(%edi), %edi
751 movdqa %xmm2, -32(%edx, %edi)
752 movdqa %xmm3, -16(%edx, %edi)
754 jl L(shl_9_end)
756 movdqa 16(%eax, %edi), %xmm2
757 sub $32, %ecx
758 movdqa 32(%eax, %edi), %xmm3
759 movdqa %xmm3, %xmm1
760 palignr $9, %xmm2, %xmm3
761 palignr $9, %xmm4, %xmm2
762 lea 32(%edi), %edi
763 movdqa %xmm2, -32(%edx, %edi)
764 movdqa %xmm3, -16(%edx, %edi)
766 jae L(shl_9_loop)
768 L(shl_9_end):
769 lea 32(%ecx), %ecx
770 add %ecx, %edi
771 add %edi, %edx
772 lea 9(%edi, %eax), %eax
773 POP (%edi)
774 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
776 ALIGN (4)
777 L(shl_10):
778 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
779 lea -10(%eax), %eax
780 movaps (%eax), %xmm1
781 xor %edi, %edi
782 lea -32(%ecx), %ecx
783 movdqu %xmm0, (%esi)
784 POP (%esi)
785 L(shl_10_loop):
787 movdqa 16(%eax, %edi), %xmm2
788 sub $32, %ecx
789 movdqa 32(%eax, %edi), %xmm3
790 movdqa %xmm3, %xmm4
791 palignr $10, %xmm2, %xmm3
792 palignr $10, %xmm1, %xmm2
793 lea 32(%edi), %edi
794 movdqa %xmm2, -32(%edx, %edi)
795 movdqa %xmm3, -16(%edx, %edi)
797 jl L(shl_10_end)
799 movdqa 16(%eax, %edi), %xmm2
800 sub $32, %ecx
801 movdqa 32(%eax, %edi), %xmm3
802 movdqa %xmm3, %xmm1
803 palignr $10, %xmm2, %xmm3
804 palignr $10, %xmm4, %xmm2
805 lea 32(%edi), %edi
806 movdqa %xmm2, -32(%edx, %edi)
807 movdqa %xmm3, -16(%edx, %edi)
809 jae L(shl_10_loop)
811 L(shl_10_end):
812 lea 32(%ecx), %ecx
813 add %ecx, %edi
814 add %edi, %edx
815 lea 10(%edi, %eax), %eax
816 POP (%edi)
817 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
819 ALIGN (4)
820 L(shl_11):
821 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
822 lea -11(%eax), %eax
823 movaps (%eax), %xmm1
824 xor %edi, %edi
825 lea -32(%ecx), %ecx
826 movdqu %xmm0, (%esi)
827 POP (%esi)
828 L(shl_11_loop):
830 movdqa 16(%eax, %edi), %xmm2
831 sub $32, %ecx
832 movdqa 32(%eax, %edi), %xmm3
833 movdqa %xmm3, %xmm4
834 palignr $11, %xmm2, %xmm3
835 palignr $11, %xmm1, %xmm2
836 lea 32(%edi), %edi
837 movdqa %xmm2, -32(%edx, %edi)
838 movdqa %xmm3, -16(%edx, %edi)
840 jl L(shl_11_end)
842 movdqa 16(%eax, %edi), %xmm2
843 sub $32, %ecx
844 movdqa 32(%eax, %edi), %xmm3
845 movdqa %xmm3, %xmm1
846 palignr $11, %xmm2, %xmm3
847 palignr $11, %xmm4, %xmm2
848 lea 32(%edi), %edi
849 movdqa %xmm2, -32(%edx, %edi)
850 movdqa %xmm3, -16(%edx, %edi)
852 jae L(shl_11_loop)
854 L(shl_11_end):
855 lea 32(%ecx), %ecx
856 add %ecx, %edi
857 add %edi, %edx
858 lea 11(%edi, %eax), %eax
859 POP (%edi)
860 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
862 ALIGN (4)
863 L(shl_12):
864 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
865 lea -12(%eax), %eax
866 movaps (%eax), %xmm1
867 xor %edi, %edi
868 lea -32(%ecx), %ecx
869 movdqu %xmm0, (%esi)
870 POP (%esi)
871 L(shl_12_loop):
873 movdqa 16(%eax, %edi), %xmm2
874 sub $32, %ecx
875 movdqa 32(%eax, %edi), %xmm3
876 movdqa %xmm3, %xmm4
877 palignr $12, %xmm2, %xmm3
878 palignr $12, %xmm1, %xmm2
879 lea 32(%edi), %edi
880 movdqa %xmm2, -32(%edx, %edi)
881 movdqa %xmm3, -16(%edx, %edi)
883 jl L(shl_12_end)
885 movdqa 16(%eax, %edi), %xmm2
886 sub $32, %ecx
887 movdqa 32(%eax, %edi), %xmm3
888 movdqa %xmm3, %xmm1
889 palignr $12, %xmm2, %xmm3
890 palignr $12, %xmm4, %xmm2
891 lea 32(%edi), %edi
892 movdqa %xmm2, -32(%edx, %edi)
893 movdqa %xmm3, -16(%edx, %edi)
895 jae L(shl_12_loop)
897 L(shl_12_end):
898 lea 32(%ecx), %ecx
899 add %ecx, %edi
900 add %edi, %edx
901 lea 12(%edi, %eax), %eax
902 POP (%edi)
903 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
905 ALIGN (4)
906 L(shl_13):
907 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
908 lea -13(%eax), %eax
909 movaps (%eax), %xmm1
910 xor %edi, %edi
911 lea -32(%ecx), %ecx
912 movdqu %xmm0, (%esi)
913 POP (%esi)
914 L(shl_13_loop):
916 movdqa 16(%eax, %edi), %xmm2
917 sub $32, %ecx
918 movdqa 32(%eax, %edi), %xmm3
919 movdqa %xmm3, %xmm4
920 palignr $13, %xmm2, %xmm3
921 palignr $13, %xmm1, %xmm2
922 lea 32(%edi), %edi
923 movdqa %xmm2, -32(%edx, %edi)
924 movdqa %xmm3, -16(%edx, %edi)
926 jl L(shl_13_end)
928 movdqa 16(%eax, %edi), %xmm2
929 sub $32, %ecx
930 movdqa 32(%eax, %edi), %xmm3
931 movdqa %xmm3, %xmm1
932 palignr $13, %xmm2, %xmm3
933 palignr $13, %xmm4, %xmm2
934 lea 32(%edi), %edi
935 movdqa %xmm2, -32(%edx, %edi)
936 movdqa %xmm3, -16(%edx, %edi)
938 jae L(shl_13_loop)
940 L(shl_13_end):
941 lea 32(%ecx), %ecx
942 add %ecx, %edi
943 add %edi, %edx
944 lea 13(%edi, %eax), %eax
945 POP (%edi)
946 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
948 ALIGN (4)
949 L(shl_14):
950 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
951 lea -14(%eax), %eax
952 movaps (%eax), %xmm1
953 xor %edi, %edi
954 lea -32(%ecx), %ecx
955 movdqu %xmm0, (%esi)
956 POP (%esi)
957 L(shl_14_loop):
959 movdqa 16(%eax, %edi), %xmm2
960 sub $32, %ecx
961 movdqa 32(%eax, %edi), %xmm3
962 movdqa %xmm3, %xmm4
963 palignr $14, %xmm2, %xmm3
964 palignr $14, %xmm1, %xmm2
965 lea 32(%edi), %edi
966 movdqa %xmm2, -32(%edx, %edi)
967 movdqa %xmm3, -16(%edx, %edi)
969 jl L(shl_14_end)
971 movdqa 16(%eax, %edi), %xmm2
972 sub $32, %ecx
973 movdqa 32(%eax, %edi), %xmm3
974 movdqa %xmm3, %xmm1
975 palignr $14, %xmm2, %xmm3
976 palignr $14, %xmm4, %xmm2
977 lea 32(%edi), %edi
978 movdqa %xmm2, -32(%edx, %edi)
979 movdqa %xmm3, -16(%edx, %edi)
981 jae L(shl_14_loop)
983 L(shl_14_end):
984 lea 32(%ecx), %ecx
985 add %ecx, %edi
986 add %edi, %edx
987 lea 14(%edi, %eax), %eax
988 POP (%edi)
989 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
992 ALIGN (4)
993 L(shl_15):
994 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
995 lea -15(%eax), %eax
996 movaps (%eax), %xmm1
997 xor %edi, %edi
998 lea -32(%ecx), %ecx
999 movdqu %xmm0, (%esi)
1000 POP (%esi)
1001 L(shl_15_loop):
1003 movdqa 16(%eax, %edi), %xmm2
1004 sub $32, %ecx
1005 movdqa 32(%eax, %edi), %xmm3
1006 movdqa %xmm3, %xmm4
1007 palignr $15, %xmm2, %xmm3
1008 palignr $15, %xmm1, %xmm2
1009 lea 32(%edi), %edi
1010 movdqa %xmm2, -32(%edx, %edi)
1011 movdqa %xmm3, -16(%edx, %edi)
1013 jl L(shl_15_end)
1015 movdqa 16(%eax, %edi), %xmm2
1016 sub $32, %ecx
1017 movdqa 32(%eax, %edi), %xmm3
1018 movdqa %xmm3, %xmm1
1019 palignr $15, %xmm2, %xmm3
1020 palignr $15, %xmm4, %xmm2
1021 lea 32(%edi), %edi
1022 movdqa %xmm2, -32(%edx, %edi)
1023 movdqa %xmm3, -16(%edx, %edi)
1025 jae L(shl_15_loop)
1027 L(shl_15_end):
1028 lea 32(%ecx), %ecx
1029 add %ecx, %edi
1030 add %edi, %edx
1031 lea 15(%edi, %eax), %eax
1032 POP (%edi)
1033 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
1036 ALIGN (4)
1037 L(fwd_write_44bytes):
1038 movl -44(%eax), %ecx
1039 movl %ecx, -44(%edx)
1040 L(fwd_write_40bytes):
1041 movl -40(%eax), %ecx
1042 movl %ecx, -40(%edx)
1043 L(fwd_write_36bytes):
1044 movl -36(%eax), %ecx
1045 movl %ecx, -36(%edx)
1046 L(fwd_write_32bytes):
1047 movl -32(%eax), %ecx
1048 movl %ecx, -32(%edx)
1049 L(fwd_write_28bytes):
1050 movl -28(%eax), %ecx
1051 movl %ecx, -28(%edx)
1052 L(fwd_write_24bytes):
1053 movl -24(%eax), %ecx
1054 movl %ecx, -24(%edx)
1055 L(fwd_write_20bytes):
1056 movl -20(%eax), %ecx
1057 movl %ecx, -20(%edx)
1058 L(fwd_write_16bytes):
1059 movl -16(%eax), %ecx
1060 movl %ecx, -16(%edx)
1061 L(fwd_write_12bytes):
1062 movl -12(%eax), %ecx
1063 movl %ecx, -12(%edx)
1064 L(fwd_write_8bytes):
1065 movl -8(%eax), %ecx
1066 movl %ecx, -8(%edx)
1067 L(fwd_write_4bytes):
1068 movl -4(%eax), %ecx
1069 movl %ecx, -4(%edx)
1070 L(fwd_write_0bytes):
1071 #ifndef USE_AS_BCOPY
1072 # ifdef USE_AS_MEMPCPY
1073 movl %edx, %eax
1074 # else
1075 movl DEST(%esp), %eax
1076 # endif
1077 #endif
1078 RETURN
1080 ALIGN (4)
1081 L(fwd_write_5bytes):
1082 movl -5(%eax), %ecx
1083 movl -4(%eax), %eax
1084 movl %ecx, -5(%edx)
1085 movl %eax, -4(%edx)
1086 #ifndef USE_AS_BCOPY
1087 # ifdef USE_AS_MEMPCPY
1088 movl %edx, %eax
1089 # else
1090 movl DEST(%esp), %eax
1091 # endif
1092 #endif
1093 RETURN
1095 ALIGN (4)
1096 L(fwd_write_45bytes):
1097 movl -45(%eax), %ecx
1098 movl %ecx, -45(%edx)
1099 L(fwd_write_41bytes):
1100 movl -41(%eax), %ecx
1101 movl %ecx, -41(%edx)
1102 L(fwd_write_37bytes):
1103 movl -37(%eax), %ecx
1104 movl %ecx, -37(%edx)
1105 L(fwd_write_33bytes):
1106 movl -33(%eax), %ecx
1107 movl %ecx, -33(%edx)
1108 L(fwd_write_29bytes):
1109 movl -29(%eax), %ecx
1110 movl %ecx, -29(%edx)
1111 L(fwd_write_25bytes):
1112 movl -25(%eax), %ecx
1113 movl %ecx, -25(%edx)
1114 L(fwd_write_21bytes):
1115 movl -21(%eax), %ecx
1116 movl %ecx, -21(%edx)
1117 L(fwd_write_17bytes):
1118 movl -17(%eax), %ecx
1119 movl %ecx, -17(%edx)
1120 L(fwd_write_13bytes):
1121 movl -13(%eax), %ecx
1122 movl %ecx, -13(%edx)
1123 L(fwd_write_9bytes):
1124 movl -9(%eax), %ecx
1125 movl %ecx, -9(%edx)
1126 movl -5(%eax), %ecx
1127 movl %ecx, -5(%edx)
1128 L(fwd_write_1bytes):
1129 movzbl -1(%eax), %ecx
1130 movb %cl, -1(%edx)
1131 #ifndef USE_AS_BCOPY
1132 # ifdef USE_AS_MEMPCPY
1133 movl %edx, %eax
1134 # else
1135 movl DEST(%esp), %eax
1136 # endif
1137 #endif
1138 RETURN
1140 ALIGN (4)
1141 L(fwd_write_46bytes):
1142 movl -46(%eax), %ecx
1143 movl %ecx, -46(%edx)
1144 L(fwd_write_42bytes):
1145 movl -42(%eax), %ecx
1146 movl %ecx, -42(%edx)
1147 L(fwd_write_38bytes):
1148 movl -38(%eax), %ecx
1149 movl %ecx, -38(%edx)
1150 L(fwd_write_34bytes):
1151 movl -34(%eax), %ecx
1152 movl %ecx, -34(%edx)
1153 L(fwd_write_30bytes):
1154 movl -30(%eax), %ecx
1155 movl %ecx, -30(%edx)
1156 L(fwd_write_26bytes):
1157 movl -26(%eax), %ecx
1158 movl %ecx, -26(%edx)
1159 L(fwd_write_22bytes):
1160 movl -22(%eax), %ecx
1161 movl %ecx, -22(%edx)
1162 L(fwd_write_18bytes):
1163 movl -18(%eax), %ecx
1164 movl %ecx, -18(%edx)
1165 L(fwd_write_14bytes):
1166 movl -14(%eax), %ecx
1167 movl %ecx, -14(%edx)
1168 L(fwd_write_10bytes):
1169 movl -10(%eax), %ecx
1170 movl %ecx, -10(%edx)
1171 L(fwd_write_6bytes):
1172 movl -6(%eax), %ecx
1173 movl %ecx, -6(%edx)
1174 L(fwd_write_2bytes):
1175 movzwl -2(%eax), %ecx
1176 movw %cx, -2(%edx)
1177 #ifndef USE_AS_BCOPY
1178 # ifdef USE_AS_MEMPCPY
1179 movl %edx, %eax
1180 # else
1181 movl DEST(%esp), %eax
1182 # endif
1183 #endif
1184 RETURN
1186 ALIGN (4)
1187 L(fwd_write_47bytes):
1188 movl -47(%eax), %ecx
1189 movl %ecx, -47(%edx)
1190 L(fwd_write_43bytes):
1191 movl -43(%eax), %ecx
1192 movl %ecx, -43(%edx)
1193 L(fwd_write_39bytes):
1194 movl -39(%eax), %ecx
1195 movl %ecx, -39(%edx)
1196 L(fwd_write_35bytes):
1197 movl -35(%eax), %ecx
1198 movl %ecx, -35(%edx)
1199 L(fwd_write_31bytes):
1200 movl -31(%eax), %ecx
1201 movl %ecx, -31(%edx)
1202 L(fwd_write_27bytes):
1203 movl -27(%eax), %ecx
1204 movl %ecx, -27(%edx)
1205 L(fwd_write_23bytes):
1206 movl -23(%eax), %ecx
1207 movl %ecx, -23(%edx)
1208 L(fwd_write_19bytes):
1209 movl -19(%eax), %ecx
1210 movl %ecx, -19(%edx)
1211 L(fwd_write_15bytes):
1212 movl -15(%eax), %ecx
1213 movl %ecx, -15(%edx)
1214 L(fwd_write_11bytes):
1215 movl -11(%eax), %ecx
1216 movl %ecx, -11(%edx)
1217 L(fwd_write_7bytes):
1218 movl -7(%eax), %ecx
1219 movl %ecx, -7(%edx)
1220 L(fwd_write_3bytes):
1221 movzwl -3(%eax), %ecx
1222 movzbl -1(%eax), %eax
1223 movw %cx, -3(%edx)
1224 movb %al, -1(%edx)
1225 #ifndef USE_AS_BCOPY
1226 # ifdef USE_AS_MEMPCPY
1227 movl %edx, %eax
1228 # else
1229 movl DEST(%esp), %eax
1230 # endif
1231 #endif
1232 RETURN
1234 ALIGN (4)
1235 L(large_page):
1236 movdqu (%eax), %xmm1
1237 lea 16(%eax), %eax
1238 movdqu %xmm0, (%esi)
1239 movntdq %xmm1, (%edx)
1240 lea 16(%edx), %edx
1241 POP (%esi)
1242 lea -0x90(%ecx), %ecx
1243 POP (%edi)
1244 L(large_page_loop):
1245 movdqu (%eax), %xmm0
1246 movdqu 0x10(%eax), %xmm1
1247 movdqu 0x20(%eax), %xmm2
1248 movdqu 0x30(%eax), %xmm3
1249 movdqu 0x40(%eax), %xmm4
1250 movdqu 0x50(%eax), %xmm5
1251 movdqu 0x60(%eax), %xmm6
1252 movdqu 0x70(%eax), %xmm7
1253 lea 0x80(%eax), %eax
1255 sub $0x80, %ecx
1256 movntdq %xmm0, (%edx)
1257 movntdq %xmm1, 0x10(%edx)
1258 movntdq %xmm2, 0x20(%edx)
1259 movntdq %xmm3, 0x30(%edx)
1260 movntdq %xmm4, 0x40(%edx)
1261 movntdq %xmm5, 0x50(%edx)
1262 movntdq %xmm6, 0x60(%edx)
1263 movntdq %xmm7, 0x70(%edx)
1264 lea 0x80(%edx), %edx
1265 jae L(large_page_loop)
1266 cmp $-0x40, %ecx
1267 lea 0x80(%ecx), %ecx
1268 jl L(large_page_less_64bytes)
1270 movdqu (%eax), %xmm0
1271 movdqu 0x10(%eax), %xmm1
1272 movdqu 0x20(%eax), %xmm2
1273 movdqu 0x30(%eax), %xmm3
1274 lea 0x40(%eax), %eax
1276 movntdq %xmm0, (%edx)
1277 movntdq %xmm1, 0x10(%edx)
1278 movntdq %xmm2, 0x20(%edx)
1279 movntdq %xmm3, 0x30(%edx)
1280 lea 0x40(%edx), %edx
1281 sub $0x40, %ecx
1282 L(large_page_less_64bytes):
1283 cmp $32, %ecx
1284 jl L(large_page_less_32bytes)
1285 movdqu (%eax), %xmm0
1286 movdqu 0x10(%eax), %xmm1
1287 lea 0x20(%eax), %eax
1288 movntdq %xmm0, (%edx)
1289 movntdq %xmm1, 0x10(%edx)
1290 lea 0x20(%edx), %edx
1291 sub $0x20, %ecx
1292 L(large_page_less_32bytes):
1293 add %ecx, %edx
1294 add %ecx, %eax
1295 sfence
1296 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
1299 ALIGN (4)
1300 L(bk_write_44bytes):
1301 movl 40(%eax), %ecx
1302 movl %ecx, 40(%edx)
1303 L(bk_write_40bytes):
1304 movl 36(%eax), %ecx
1305 movl %ecx, 36(%edx)
1306 L(bk_write_36bytes):
1307 movl 32(%eax), %ecx
1308 movl %ecx, 32(%edx)
1309 L(bk_write_32bytes):
1310 movl 28(%eax), %ecx
1311 movl %ecx, 28(%edx)
1312 L(bk_write_28bytes):
1313 movl 24(%eax), %ecx
1314 movl %ecx, 24(%edx)
1315 L(bk_write_24bytes):
1316 movl 20(%eax), %ecx
1317 movl %ecx, 20(%edx)
1318 L(bk_write_20bytes):
1319 movl 16(%eax), %ecx
1320 movl %ecx, 16(%edx)
1321 L(bk_write_16bytes):
1322 movl 12(%eax), %ecx
1323 movl %ecx, 12(%edx)
1324 L(bk_write_12bytes):
1325 movl 8(%eax), %ecx
1326 movl %ecx, 8(%edx)
1327 L(bk_write_8bytes):
1328 movl 4(%eax), %ecx
1329 movl %ecx, 4(%edx)
1330 L(bk_write_4bytes):
1331 movl (%eax), %ecx
1332 movl %ecx, (%edx)
1333 L(bk_write_0bytes):
1334 #ifndef USE_AS_BCOPY
1335 movl DEST(%esp), %eax
1336 # ifdef USE_AS_MEMPCPY
1337 movl LEN(%esp), %ecx
1338 add %ecx, %eax
1339 # endif
1340 #endif
1341 RETURN
1343 ALIGN (4)
1344 L(bk_write_45bytes):
1345 movl 41(%eax), %ecx
1346 movl %ecx, 41(%edx)
1347 L(bk_write_41bytes):
1348 movl 37(%eax), %ecx
1349 movl %ecx, 37(%edx)
1350 L(bk_write_37bytes):
1351 movl 33(%eax), %ecx
1352 movl %ecx, 33(%edx)
1353 L(bk_write_33bytes):
1354 movl 29(%eax), %ecx
1355 movl %ecx, 29(%edx)
1356 L(bk_write_29bytes):
1357 movl 25(%eax), %ecx
1358 movl %ecx, 25(%edx)
1359 L(bk_write_25bytes):
1360 movl 21(%eax), %ecx
1361 movl %ecx, 21(%edx)
1362 L(bk_write_21bytes):
1363 movl 17(%eax), %ecx
1364 movl %ecx, 17(%edx)
1365 L(bk_write_17bytes):
1366 movl 13(%eax), %ecx
1367 movl %ecx, 13(%edx)
1368 L(bk_write_13bytes):
1369 movl 9(%eax), %ecx
1370 movl %ecx, 9(%edx)
1371 L(bk_write_9bytes):
1372 movl 5(%eax), %ecx
1373 movl %ecx, 5(%edx)
1374 L(bk_write_5bytes):
1375 movl 1(%eax), %ecx
1376 movl %ecx, 1(%edx)
1377 L(bk_write_1bytes):
1378 movzbl (%eax), %ecx
1379 movb %cl, (%edx)
1380 #ifndef USE_AS_BCOPY
1381 movl DEST(%esp), %eax
1382 # ifdef USE_AS_MEMPCPY
1383 movl LEN(%esp), %ecx
1384 add %ecx, %eax
1385 # endif
1386 #endif
1387 RETURN
1389 ALIGN (4)
1390 L(bk_write_46bytes):
1391 movl 42(%eax), %ecx
1392 movl %ecx, 42(%edx)
1393 L(bk_write_42bytes):
1394 movl 38(%eax), %ecx
1395 movl %ecx, 38(%edx)
1396 L(bk_write_38bytes):
1397 movl 34(%eax), %ecx
1398 movl %ecx, 34(%edx)
1399 L(bk_write_34bytes):
1400 movl 30(%eax), %ecx
1401 movl %ecx, 30(%edx)
1402 L(bk_write_30bytes):
1403 movl 26(%eax), %ecx
1404 movl %ecx, 26(%edx)
1405 L(bk_write_26bytes):
1406 movl 22(%eax), %ecx
1407 movl %ecx, 22(%edx)
1408 L(bk_write_22bytes):
1409 movl 18(%eax), %ecx
1410 movl %ecx, 18(%edx)
1411 L(bk_write_18bytes):
1412 movl 14(%eax), %ecx
1413 movl %ecx, 14(%edx)
1414 L(bk_write_14bytes):
1415 movl 10(%eax), %ecx
1416 movl %ecx, 10(%edx)
1417 L(bk_write_10bytes):
1418 movl 6(%eax), %ecx
1419 movl %ecx, 6(%edx)
1420 L(bk_write_6bytes):
1421 movl 2(%eax), %ecx
1422 movl %ecx, 2(%edx)
1423 L(bk_write_2bytes):
1424 movzwl (%eax), %ecx
1425 movw %cx, (%edx)
1426 #ifndef USE_AS_BCOPY
1427 movl DEST(%esp), %eax
1428 # ifdef USE_AS_MEMPCPY
1429 movl LEN(%esp), %ecx
1430 add %ecx, %eax
1431 # endif
1432 #endif
1433 RETURN
1435 ALIGN (4)
1436 L(bk_write_47bytes):
1437 movl 43(%eax), %ecx
1438 movl %ecx, 43(%edx)
1439 L(bk_write_43bytes):
1440 movl 39(%eax), %ecx
1441 movl %ecx, 39(%edx)
1442 L(bk_write_39bytes):
1443 movl 35(%eax), %ecx
1444 movl %ecx, 35(%edx)
1445 L(bk_write_35bytes):
1446 movl 31(%eax), %ecx
1447 movl %ecx, 31(%edx)
1448 L(bk_write_31bytes):
1449 movl 27(%eax), %ecx
1450 movl %ecx, 27(%edx)
1451 L(bk_write_27bytes):
1452 movl 23(%eax), %ecx
1453 movl %ecx, 23(%edx)
1454 L(bk_write_23bytes):
1455 movl 19(%eax), %ecx
1456 movl %ecx, 19(%edx)
1457 L(bk_write_19bytes):
1458 movl 15(%eax), %ecx
1459 movl %ecx, 15(%edx)
1460 L(bk_write_15bytes):
1461 movl 11(%eax), %ecx
1462 movl %ecx, 11(%edx)
1463 L(bk_write_11bytes):
1464 movl 7(%eax), %ecx
1465 movl %ecx, 7(%edx)
1466 L(bk_write_7bytes):
1467 movl 3(%eax), %ecx
1468 movl %ecx, 3(%edx)
1469 L(bk_write_3bytes):
1470 movzwl 1(%eax), %ecx
1471 movw %cx, 1(%edx)
1472 movzbl (%eax), %eax
1473 movb %al, (%edx)
1474 #ifndef USE_AS_BCOPY
1475 movl DEST(%esp), %eax
1476 # ifdef USE_AS_MEMPCPY
1477 movl LEN(%esp), %ecx
1478 add %ecx, %eax
1479 # endif
1480 #endif
1481 RETURN_END
1484 .pushsection .rodata.ssse3,"a",@progbits
1485 ALIGN (2)
1486 L(table_48bytes_fwd):
1487 .int JMPTBL (L(fwd_write_0bytes), L(table_48bytes_fwd))
1488 .int JMPTBL (L(fwd_write_1bytes), L(table_48bytes_fwd))
1489 .int JMPTBL (L(fwd_write_2bytes), L(table_48bytes_fwd))
1490 .int JMPTBL (L(fwd_write_3bytes), L(table_48bytes_fwd))
1491 .int JMPTBL (L(fwd_write_4bytes), L(table_48bytes_fwd))
1492 .int JMPTBL (L(fwd_write_5bytes), L(table_48bytes_fwd))
1493 .int JMPTBL (L(fwd_write_6bytes), L(table_48bytes_fwd))
1494 .int JMPTBL (L(fwd_write_7bytes), L(table_48bytes_fwd))
1495 .int JMPTBL (L(fwd_write_8bytes), L(table_48bytes_fwd))
1496 .int JMPTBL (L(fwd_write_9bytes), L(table_48bytes_fwd))
1497 .int JMPTBL (L(fwd_write_10bytes), L(table_48bytes_fwd))
1498 .int JMPTBL (L(fwd_write_11bytes), L(table_48bytes_fwd))
1499 .int JMPTBL (L(fwd_write_12bytes), L(table_48bytes_fwd))
1500 .int JMPTBL (L(fwd_write_13bytes), L(table_48bytes_fwd))
1501 .int JMPTBL (L(fwd_write_14bytes), L(table_48bytes_fwd))
1502 .int JMPTBL (L(fwd_write_15bytes), L(table_48bytes_fwd))
1503 .int JMPTBL (L(fwd_write_16bytes), L(table_48bytes_fwd))
1504 .int JMPTBL (L(fwd_write_17bytes), L(table_48bytes_fwd))
1505 .int JMPTBL (L(fwd_write_18bytes), L(table_48bytes_fwd))
1506 .int JMPTBL (L(fwd_write_19bytes), L(table_48bytes_fwd))
1507 .int JMPTBL (L(fwd_write_20bytes), L(table_48bytes_fwd))
1508 .int JMPTBL (L(fwd_write_21bytes), L(table_48bytes_fwd))
1509 .int JMPTBL (L(fwd_write_22bytes), L(table_48bytes_fwd))
1510 .int JMPTBL (L(fwd_write_23bytes), L(table_48bytes_fwd))
1511 .int JMPTBL (L(fwd_write_24bytes), L(table_48bytes_fwd))
1512 .int JMPTBL (L(fwd_write_25bytes), L(table_48bytes_fwd))
1513 .int JMPTBL (L(fwd_write_26bytes), L(table_48bytes_fwd))
1514 .int JMPTBL (L(fwd_write_27bytes), L(table_48bytes_fwd))
1515 .int JMPTBL (L(fwd_write_28bytes), L(table_48bytes_fwd))
1516 .int JMPTBL (L(fwd_write_29bytes), L(table_48bytes_fwd))
1517 .int JMPTBL (L(fwd_write_30bytes), L(table_48bytes_fwd))
1518 .int JMPTBL (L(fwd_write_31bytes), L(table_48bytes_fwd))
1519 .int JMPTBL (L(fwd_write_32bytes), L(table_48bytes_fwd))
1520 .int JMPTBL (L(fwd_write_33bytes), L(table_48bytes_fwd))
1521 .int JMPTBL (L(fwd_write_34bytes), L(table_48bytes_fwd))
1522 .int JMPTBL (L(fwd_write_35bytes), L(table_48bytes_fwd))
1523 .int JMPTBL (L(fwd_write_36bytes), L(table_48bytes_fwd))
1524 .int JMPTBL (L(fwd_write_37bytes), L(table_48bytes_fwd))
1525 .int JMPTBL (L(fwd_write_38bytes), L(table_48bytes_fwd))
1526 .int JMPTBL (L(fwd_write_39bytes), L(table_48bytes_fwd))
1527 .int JMPTBL (L(fwd_write_40bytes), L(table_48bytes_fwd))
1528 .int JMPTBL (L(fwd_write_41bytes), L(table_48bytes_fwd))
1529 .int JMPTBL (L(fwd_write_42bytes), L(table_48bytes_fwd))
1530 .int JMPTBL (L(fwd_write_43bytes), L(table_48bytes_fwd))
1531 .int JMPTBL (L(fwd_write_44bytes), L(table_48bytes_fwd))
1532 .int JMPTBL (L(fwd_write_45bytes), L(table_48bytes_fwd))
1533 .int JMPTBL (L(fwd_write_46bytes), L(table_48bytes_fwd))
1534 .int JMPTBL (L(fwd_write_47bytes), L(table_48bytes_fwd))
1536 ALIGN (2)
1537 L(shl_table):
1538 .int JMPTBL (L(shl_0), L(shl_table))
1539 .int JMPTBL (L(shl_1), L(shl_table))
1540 .int JMPTBL (L(shl_2), L(shl_table))
1541 .int JMPTBL (L(shl_3), L(shl_table))
1542 .int JMPTBL (L(shl_4), L(shl_table))
1543 .int JMPTBL (L(shl_5), L(shl_table))
1544 .int JMPTBL (L(shl_6), L(shl_table))
1545 .int JMPTBL (L(shl_7), L(shl_table))
1546 .int JMPTBL (L(shl_8), L(shl_table))
1547 .int JMPTBL (L(shl_9), L(shl_table))
1548 .int JMPTBL (L(shl_10), L(shl_table))
1549 .int JMPTBL (L(shl_11), L(shl_table))
1550 .int JMPTBL (L(shl_12), L(shl_table))
1551 .int JMPTBL (L(shl_13), L(shl_table))
1552 .int JMPTBL (L(shl_14), L(shl_table))
1553 .int JMPTBL (L(shl_15), L(shl_table))
1555 ALIGN (2)
1556 L(table_48_bytes_bwd):
1557 .int JMPTBL (L(bk_write_0bytes), L(table_48_bytes_bwd))
1558 .int JMPTBL (L(bk_write_1bytes), L(table_48_bytes_bwd))
1559 .int JMPTBL (L(bk_write_2bytes), L(table_48_bytes_bwd))
1560 .int JMPTBL (L(bk_write_3bytes), L(table_48_bytes_bwd))
1561 .int JMPTBL (L(bk_write_4bytes), L(table_48_bytes_bwd))
1562 .int JMPTBL (L(bk_write_5bytes), L(table_48_bytes_bwd))
1563 .int JMPTBL (L(bk_write_6bytes), L(table_48_bytes_bwd))
1564 .int JMPTBL (L(bk_write_7bytes), L(table_48_bytes_bwd))
1565 .int JMPTBL (L(bk_write_8bytes), L(table_48_bytes_bwd))
1566 .int JMPTBL (L(bk_write_9bytes), L(table_48_bytes_bwd))
1567 .int JMPTBL (L(bk_write_10bytes), L(table_48_bytes_bwd))
1568 .int JMPTBL (L(bk_write_11bytes), L(table_48_bytes_bwd))
1569 .int JMPTBL (L(bk_write_12bytes), L(table_48_bytes_bwd))
1570 .int JMPTBL (L(bk_write_13bytes), L(table_48_bytes_bwd))
1571 .int JMPTBL (L(bk_write_14bytes), L(table_48_bytes_bwd))
1572 .int JMPTBL (L(bk_write_15bytes), L(table_48_bytes_bwd))
1573 .int JMPTBL (L(bk_write_16bytes), L(table_48_bytes_bwd))
1574 .int JMPTBL (L(bk_write_17bytes), L(table_48_bytes_bwd))
1575 .int JMPTBL (L(bk_write_18bytes), L(table_48_bytes_bwd))
1576 .int JMPTBL (L(bk_write_19bytes), L(table_48_bytes_bwd))
1577 .int JMPTBL (L(bk_write_20bytes), L(table_48_bytes_bwd))
1578 .int JMPTBL (L(bk_write_21bytes), L(table_48_bytes_bwd))
1579 .int JMPTBL (L(bk_write_22bytes), L(table_48_bytes_bwd))
1580 .int JMPTBL (L(bk_write_23bytes), L(table_48_bytes_bwd))
1581 .int JMPTBL (L(bk_write_24bytes), L(table_48_bytes_bwd))
1582 .int JMPTBL (L(bk_write_25bytes), L(table_48_bytes_bwd))
1583 .int JMPTBL (L(bk_write_26bytes), L(table_48_bytes_bwd))
1584 .int JMPTBL (L(bk_write_27bytes), L(table_48_bytes_bwd))
1585 .int JMPTBL (L(bk_write_28bytes), L(table_48_bytes_bwd))
1586 .int JMPTBL (L(bk_write_29bytes), L(table_48_bytes_bwd))
1587 .int JMPTBL (L(bk_write_30bytes), L(table_48_bytes_bwd))
1588 .int JMPTBL (L(bk_write_31bytes), L(table_48_bytes_bwd))
1589 .int JMPTBL (L(bk_write_32bytes), L(table_48_bytes_bwd))
1590 .int JMPTBL (L(bk_write_33bytes), L(table_48_bytes_bwd))
1591 .int JMPTBL (L(bk_write_34bytes), L(table_48_bytes_bwd))
1592 .int JMPTBL (L(bk_write_35bytes), L(table_48_bytes_bwd))
1593 .int JMPTBL (L(bk_write_36bytes), L(table_48_bytes_bwd))
1594 .int JMPTBL (L(bk_write_37bytes), L(table_48_bytes_bwd))
1595 .int JMPTBL (L(bk_write_38bytes), L(table_48_bytes_bwd))
1596 .int JMPTBL (L(bk_write_39bytes), L(table_48_bytes_bwd))
1597 .int JMPTBL (L(bk_write_40bytes), L(table_48_bytes_bwd))
1598 .int JMPTBL (L(bk_write_41bytes), L(table_48_bytes_bwd))
1599 .int JMPTBL (L(bk_write_42bytes), L(table_48_bytes_bwd))
1600 .int JMPTBL (L(bk_write_43bytes), L(table_48_bytes_bwd))
1601 .int JMPTBL (L(bk_write_44bytes), L(table_48_bytes_bwd))
1602 .int JMPTBL (L(bk_write_45bytes), L(table_48_bytes_bwd))
1603 .int JMPTBL (L(bk_write_46bytes), L(table_48_bytes_bwd))
1604 .int JMPTBL (L(bk_write_47bytes), L(table_48_bytes_bwd))
1606 .popsection
1608 #ifdef USE_AS_MEMMOVE
1609 ALIGN (4)
1610 L(copy_backward):
1611 PUSH (%esi)
1612 movl %eax, %esi
1613 lea (%ecx,%edx,1),%edx
1614 lea (%ecx,%esi,1),%esi
1615 testl $0x3, %edx
1616 jnz L(bk_align)
1618 L(bk_aligned_4):
1619 cmp $64, %ecx
1620 jge L(bk_write_more64bytes)
1622 L(bk_write_64bytesless):
1623 cmp $32, %ecx
1624 jl L(bk_write_less32bytes)
1626 L(bk_write_more32bytes):
1627 /* Copy 32 bytes at a time. */
1628 sub $32, %ecx
1629 movl -4(%esi), %eax
1630 movl %eax, -4(%edx)
1631 movl -8(%esi), %eax
1632 movl %eax, -8(%edx)
1633 movl -12(%esi), %eax
1634 movl %eax, -12(%edx)
1635 movl -16(%esi), %eax
1636 movl %eax, -16(%edx)
1637 movl -20(%esi), %eax
1638 movl %eax, -20(%edx)
1639 movl -24(%esi), %eax
1640 movl %eax, -24(%edx)
1641 movl -28(%esi), %eax
1642 movl %eax, -28(%edx)
1643 movl -32(%esi), %eax
1644 movl %eax, -32(%edx)
1645 sub $32, %edx
1646 sub $32, %esi
1648 L(bk_write_less32bytes):
1649 movl %esi, %eax
1650 sub %ecx, %edx
1651 sub %ecx, %eax
1652 POP (%esi)
1653 L(bk_write_less32bytes_2):
1654 BRANCH_TO_JMPTBL_ENTRY (L(table_48_bytes_bwd), %ecx, 4)
1656 ALIGN (4)
1657 L(bk_align):
1658 cmp $8, %ecx
1659 jle L(bk_write_less32bytes)
1660 testl $1, %edx
1661 /* We get here only if (EDX & 3 ) != 0 so if (EDX & 1) ==0,
1662 then (EDX & 2) must be != 0. */
1663 jz L(bk_got2)
1664 sub $1, %esi
1665 sub $1, %ecx
1666 sub $1, %edx
1667 movzbl (%esi), %eax
1668 movb %al, (%edx)
1670 testl $2, %edx
1671 jz L(bk_aligned_4)
1673 L(bk_got2):
1674 sub $2, %esi
1675 sub $2, %ecx
1676 sub $2, %edx
1677 movzwl (%esi), %eax
1678 movw %ax, (%edx)
1679 jmp L(bk_aligned_4)
1681 ALIGN (4)
1682 L(bk_write_more64bytes):
1683 /* Check alignment of last byte. */
1684 testl $15, %edx
1685 jz L(bk_ssse3_cpy_pre)
1687 /* EDX is aligned 4 bytes, but not 16 bytes. */
1688 L(bk_ssse3_align):
1689 sub $4, %esi
1690 sub $4, %ecx
1691 sub $4, %edx
1692 movl (%esi), %eax
1693 movl %eax, (%edx)
1695 testl $15, %edx
1696 jz L(bk_ssse3_cpy_pre)
1698 sub $4, %esi
1699 sub $4, %ecx
1700 sub $4, %edx
1701 movl (%esi), %eax
1702 movl %eax, (%edx)
1704 testl $15, %edx
1705 jz L(bk_ssse3_cpy_pre)
1707 sub $4, %esi
1708 sub $4, %ecx
1709 sub $4, %edx
1710 movl (%esi), %eax
1711 movl %eax, (%edx)
1713 L(bk_ssse3_cpy_pre):
1714 cmp $64, %ecx
1715 jl L(bk_write_more32bytes)
1717 L(bk_ssse3_cpy):
1718 sub $64, %esi
1719 sub $64, %ecx
1720 sub $64, %edx
1721 movdqu 0x30(%esi), %xmm3
1722 movdqa %xmm3, 0x30(%edx)
1723 movdqu 0x20(%esi), %xmm2
1724 movdqa %xmm2, 0x20(%edx)
1725 movdqu 0x10(%esi), %xmm1
1726 movdqa %xmm1, 0x10(%edx)
1727 movdqu (%esi), %xmm0
1728 movdqa %xmm0, (%edx)
1729 cmp $64, %ecx
1730 jge L(bk_ssse3_cpy)
1731 jmp L(bk_write_64bytesless)
1733 #endif
1735 END (MEMCPY)
1737 #endif