| blob | b26037d279f813ec12fc5faa781413f352c47e9c |
1 /* memcpy with SSSE3 and REP string.
2 Copyright (C) 2010 Free Software Foundation, Inc.
3 Contributed by Intel Corporation.
4 This file is part of the GNU C Library.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, write to the Free
18 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
19 02111-1307 USA. */
21 #include <sysdep.h>
23 #if !defined NOT_IN_libc \
24 && (defined SHARED \
25 || defined USE_AS_MEMMOVE \
26 || !defined USE_MULTIARCH)
28 #include "asm-syntax.h"
30 #ifndef MEMCPY
31 # define MEMCPY __memcpy_ssse3_rep
32 # define MEMCPY_CHK __memcpy_chk_ssse3_rep
33 #endif
35 #ifdef USE_AS_BCOPY
36 # define SRC PARMS
37 # define DEST SRC+4
38 # define LEN DEST+4
39 #else
40 # define DEST PARMS
41 # define SRC DEST+4
42 # define LEN SRC+4
43 #endif
45 #define CFI_PUSH(REG) \
46 cfi_adjust_cfa_offset (4); \
47 cfi_rel_offset (REG, 0)
49 #define CFI_POP(REG) \
50 cfi_adjust_cfa_offset (-4); \
51 cfi_restore (REG)
53 #define PUSH(REG) pushl REG; CFI_PUSH (REG)
54 #define POP(REG) popl REG; CFI_POP (REG)
56 #ifdef SHARED
57 # define PARMS 8 /* Preserve EBX. */
58 # define ENTRANCE PUSH (%ebx);
59 # define RETURN_END POP (%ebx); ret
60 # define RETURN RETURN_END; CFI_PUSH (%ebx)
61 # define JMPTBL(I, B) I - B
63 /* Load an entry in a jump table into EBX and branch to it. TABLE is a
64 jump table with relative offsets. INDEX is a register contains the
65 index into the jump table. SCALE is the scale of INDEX. */
66 # define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \
67 /* We first load PC into EBX. */ \
68 call __i686.get_pc_thunk.bx; \
69 /* Get the address of the jump table. */ \
70 addl $(TABLE - .), %ebx; \
71 /* Get the entry and convert the relative offset to the \
72 absolute address. */ \
73 addl (%ebx,INDEX,SCALE), %ebx; \
74 /* We loaded the jump table. Go. */ \
75 jmp *%ebx
77 # define BRANCH_TO_JMPTBL_ENTRY_VALUE(TABLE) \
78 addl $(TABLE - .), %ebx
80 # define BRANCH_TO_JMPTBL_ENTRY_TAIL(TABLE, INDEX, SCALE) \
81 addl (%ebx,INDEX,SCALE), %ebx; \
82 /* We loaded the jump table. Go. */ \
83 jmp *%ebx
85 .section .gnu.linkonce.t.__i686.get_pc_thunk.bx,"ax",@progbits
86 .globl __i686.get_pc_thunk.bx
87 .hidden __i686.get_pc_thunk.bx
88 ALIGN (4)
89 .type __i686.get_pc_thunk.bx,@function
90 __i686.get_pc_thunk.bx:
91 movl (%esp), %ebx
92 ret
93 #else
94 # define PARMS 4
95 # define ENTRANCE
96 # define RETURN_END ret
97 # define RETURN RETURN_END
98 # define JMPTBL(I, B) I
100 /* Branch to an entry in a jump table. TABLE is a jump table with
101 absolute offsets. INDEX is a register contains the index into the
102 jump table. SCALE is the scale of INDEX. */
103 # define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \
104 jmp *TABLE(,INDEX,SCALE)
106 # define BRANCH_TO_JMPTBL_ENTRY_VALUE(TABLE)
108 # define BRANCH_TO_JMPTBL_ENTRY_TAIL(TABLE, INDEX, SCALE) \
109 jmp *TABLE(,INDEX,SCALE)
110 #endif
112 .section .text.ssse3,"ax",@progbits
113 #if defined SHARED && !defined NOT_IN_libc && !defined USE_AS_BCOPY
114 ENTRY (MEMCPY_CHK)
115 movl 12(%esp), %eax
116 cmpl %eax, 16(%esp)
117 jb HIDDEN_JUMPTARGET (__chk_fail)
118 END (MEMCPY_CHK)
119 #endif
120 ENTRY (MEMCPY)
121 ENTRANCE
122 movl LEN(%esp), %ecx
123 movl SRC(%esp), %eax
124 movl DEST(%esp), %edx
126 #ifdef USE_AS_MEMMOVE
127 cmp %eax, %edx
128 jb L(copy_forward)
129 je L(fwd_write_0bytes)
130 cmp $32, %ecx
131 jge L(memmove_bwd)
132 jmp L(bk_write_less32bytes_2)
133 L(memmove_bwd):
134 add %ecx, %eax
135 cmp %eax, %edx
136 movl SRC(%esp), %eax
137 jb L(copy_backward)
139 L(copy_forward):
140 #endif
141 cmp $48, %ecx
142 jge L(48bytesormore)
144 L(fwd_write_less32bytes):
145 #ifndef USE_AS_MEMMOVE
146 cmp %dl, %al
147 jl L(bk_write)
148 #endif
149 add %ecx, %edx
150 add %ecx, %eax
151 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
152 #ifndef USE_AS_MEMMOVE
153 L(bk_write):
154 BRANCH_TO_JMPTBL_ENTRY (L(table_48_bytes_bwd), %ecx, 4)
155 #endif
157 ALIGN (4)
158 /* ECX > 32 and EDX is 4 byte aligned. */
159 L(48bytesormore):
160 movdqu (%eax), %xmm0
161 PUSH (%edi)
162 movl %edx, %edi
163 and $-16, %edx
164 PUSH (%esi)
165 add $16, %edx
166 movl %edi, %esi
167 sub %edx, %edi
168 add %edi, %ecx
169 sub %edi, %eax
171 #ifdef SHARED_CACHE_SIZE_HALF
172 cmp $SHARED_CACHE_SIZE_HALF, %ecx
173 #else
174 # ifdef SHARED
175 call __i686.get_pc_thunk.bx
176 add $_GLOBAL_OFFSET_TABLE_, %ebx
177 cmp __x86_shared_cache_size_half@GOTOFF(%ebx), %ecx
178 # else
179 cmp __x86_shared_cache_size_half, %ecx
180 # endif
181 #endif
183 mov %eax, %edi
184 jge L(large_page)
185 and $0xf, %edi
186 jz L(shl_0)
188 BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %edi, 4)
190 ALIGN (4)
191 L(shl_0):
192 movdqu %xmm0, (%esi)
193 xor %edi, %edi
194 cmp $127, %ecx
195 ja L(shl_0_gobble)
196 lea -32(%ecx), %ecx
197 L(shl_0_loop):
198 movdqa (%eax, %edi), %xmm0
199 movdqa 16(%eax, %edi), %xmm1
200 sub $32, %ecx
201 movdqa %xmm0, (%edx, %edi)
202 movdqa %xmm1, 16(%edx, %edi)
203 lea 32(%edi), %edi
204 jl L(shl_0_end)
206 movdqa (%eax, %edi), %xmm0
207 movdqa 16(%eax, %edi), %xmm1
208 sub $32, %ecx
209 movdqa %xmm0, (%edx, %edi)
210 movdqa %xmm1, 16(%edx, %edi)
211 lea 32(%edi), %edi
212 jl L(shl_0_end)
214 movdqa (%eax, %edi), %xmm0
215 movdqa 16(%eax, %edi), %xmm1
216 sub $32, %ecx
217 movdqa %xmm0, (%edx, %edi)
218 movdqa %xmm1, 16(%edx, %edi)
219 lea 32(%edi), %edi
220 jl L(shl_0_end)
222 movdqa (%eax, %edi), %xmm0
223 movdqa 16(%eax, %edi), %xmm1
224 sub $32, %ecx
225 movdqa %xmm0, (%edx, %edi)
226 movdqa %xmm1, 16(%edx, %edi)
227 lea 32(%edi), %edi
228 L(shl_0_end):
229 lea 32(%ecx), %ecx
230 add %ecx, %edi
231 add %edi, %edx
232 add %edi, %eax
233 POP (%esi)
234 POP (%edi)
235 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
237 L(shl_0_gobble):
239 #ifdef DATA_CACHE_SIZE_HALF
240 cmp $DATA_CACHE_SIZE_HALF, %ecx
241 #else
242 # ifdef SHARED
243 call __i686.get_pc_thunk.bx
244 add $_GLOBAL_OFFSET_TABLE_, %ebx
245 mov __x86_data_cache_size_half@GOTOFF(%ebx), %edi
246 # else
247 mov __x86_data_cache_size_half, %edi
248 # endif
249 #endif
250 mov %edi, %esi
251 shr $3, %esi
252 sub %esi, %edi
253 cmp %edi, %ecx
254 jge L(shl_0_gobble_mem_start)
255 lea -128(%ecx), %ecx
256 ALIGN (4)
257 L(shl_0_gobble_cache_loop):
258 movdqa (%eax), %xmm0
259 movaps 0x10(%eax), %xmm1
260 movaps 0x20(%eax), %xmm2
261 movaps 0x30(%eax), %xmm3
262 movaps 0x40(%eax), %xmm4
263 movaps 0x50(%eax), %xmm5
264 movaps 0x60(%eax), %xmm6
265 movaps 0x70(%eax), %xmm7
266 lea 0x80(%eax), %eax
267 sub $128, %ecx
268 movdqa %xmm0, (%edx)
269 movaps %xmm1, 0x10(%edx)
270 movaps %xmm2, 0x20(%edx)
271 movaps %xmm3, 0x30(%edx)
272 movaps %xmm4, 0x40(%edx)
273 movaps %xmm5, 0x50(%edx)
274 movaps %xmm6, 0x60(%edx)
275 movaps %xmm7, 0x70(%edx)
276 lea 0x80(%edx), %edx
278 jge L(shl_0_gobble_cache_loop)
279 L(shl_0_gobble_cache_loop_tail):
280 cmp $-0x40, %ecx
281 lea 0x80(%ecx), %ecx
282 jl L(shl_0_cache_less_64bytes)
284 movdqa (%eax), %xmm0
285 sub $0x40, %ecx
286 movdqa 0x10(%eax), %xmm1
288 movdqa %xmm0, (%edx)
289 movdqa %xmm1, 0x10(%edx)
291 movdqa 0x20(%eax), %xmm0
292 movdqa 0x30(%eax), %xmm1
293 add $0x40, %eax
295 movdqa %xmm0, 0x20(%edx)
296 movdqa %xmm1, 0x30(%edx)
297 add $0x40, %edx
298 L(shl_0_cache_less_64bytes):
299 cmp $0x20, %ecx
300 jl L(shl_0_cache_less_32bytes)
301 movdqa (%eax), %xmm0
302 sub $0x20, %ecx
303 movdqa 0x10(%eax), %xmm1
304 add $0x20, %eax
305 movdqa %xmm0, (%edx)
306 movdqa %xmm1, 0x10(%edx)
307 add $0x20, %edx
308 L(shl_0_cache_less_32bytes):
309 cmp $0x10, %ecx
310 jl L(shl_0_cache_less_16bytes)
311 sub $0x10, %ecx
312 movdqa (%eax), %xmm0
313 add $0x10, %eax
314 movdqa %xmm0, (%edx)
315 add $0x10, %edx
316 L(shl_0_cache_less_16bytes):
317 add %ecx, %edx
318 add %ecx, %eax
319 POP (%esi)
320 POP (%edi)
321 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
324 ALIGN (4)
325 L(shl_0_gobble_mem_start):
326 cmp %al, %dl
327 je L(copy_page_by_rep)
328 lea -128(%ecx), %ecx
329 L(shl_0_gobble_mem_loop):
330 prefetchnta 0x1c0(%eax)
331 prefetchnta 0x280(%eax)
332 prefetchnta 0x1c0(%edx)
333 prefetchnta 0x280(%edx)
335 movdqa (%eax), %xmm0
336 movaps 0x10(%eax), %xmm1
337 movaps 0x20(%eax), %xmm2
338 movaps 0x30(%eax), %xmm3
339 movaps 0x40(%eax), %xmm4
340 movaps 0x50(%eax), %xmm5
341 movaps 0x60(%eax), %xmm6
342 movaps 0x70(%eax), %xmm7
343 lea 0x80(%eax), %eax
344 sub $0x80, %ecx
345 movdqa %xmm0, (%edx)
346 movaps %xmm1, 0x10(%edx)
347 movaps %xmm2, 0x20(%edx)
348 movaps %xmm3, 0x30(%edx)
349 movaps %xmm4, 0x40(%edx)
350 movaps %xmm5, 0x50(%edx)
351 movaps %xmm6, 0x60(%edx)
352 movaps %xmm7, 0x70(%edx)
353 lea 0x80(%edx), %edx
355 jge L(shl_0_gobble_mem_loop)
356 cmp $-0x40, %ecx
357 lea 0x80(%ecx), %ecx
358 jl L(shl_0_mem_less_64bytes)
360 movdqa (%eax), %xmm0
361 sub $0x40, %ecx
362 movdqa 0x10(%eax), %xmm1
364 movdqa %xmm0, (%edx)
365 movdqa %xmm1, 0x10(%edx)
367 movdqa 0x20(%eax), %xmm0
368 movdqa 0x30(%eax), %xmm1
369 add $0x40, %eax
371 movdqa %xmm0, 0x20(%edx)
372 movdqa %xmm1, 0x30(%edx)
373 add $0x40, %edx
374 L(shl_0_mem_less_64bytes):
375 cmp $0x20, %ecx
376 jl L(shl_0_mem_less_32bytes)
377 movdqa (%eax), %xmm0
378 sub $0x20, %ecx
379 movdqa 0x10(%eax), %xmm1
380 add $0x20, %eax
381 movdqa %xmm0, (%edx)
382 movdqa %xmm1, 0x10(%edx)
383 add $0x20, %edx
384 L(shl_0_mem_less_32bytes):
385 cmp $0x10, %ecx
386 jl L(shl_0_mem_less_16bytes)
387 sub $0x10, %ecx
388 movdqa (%eax), %xmm0
389 add $0x10, %eax
390 movdqa %xmm0, (%edx)
391 add $0x10, %edx
392 L(shl_0_mem_less_16bytes):
393 add %ecx, %edx
394 add %ecx, %eax
395 POP (%esi)
396 POP (%edi)
397 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
400 ALIGN (4)
401 L(shl_1):
402 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
403 lea -1(%eax), %eax
404 movaps (%eax), %xmm1
405 xor %edi, %edi
406 lea -32(%ecx), %ecx
407 movdqu %xmm0, (%esi)
408 POP (%esi)
409 L(shl_1_loop):
411 movdqa 16(%eax, %edi), %xmm2
412 sub $32, %ecx
413 movdqa 32(%eax, %edi), %xmm3
414 movdqa %xmm3, %xmm4
415 palignr $1, %xmm2, %xmm3
416 palignr $1, %xmm1, %xmm2
417 lea 32(%edi), %edi
418 movdqa %xmm2, -32(%edx, %edi)
419 movdqa %xmm3, -16(%edx, %edi)
421 jl L(shl_1_end)
423 movdqa 16(%eax, %edi), %xmm2
424 sub $32, %ecx
425 movdqa 32(%eax, %edi), %xmm3
426 movdqa %xmm3, %xmm1
427 palignr $1, %xmm2, %xmm3
428 palignr $1, %xmm4, %xmm2
429 lea 32(%edi), %edi
430 movdqa %xmm2, -32(%edx, %edi)
431 movdqa %xmm3, -16(%edx, %edi)
433 jae L(shl_1_loop)
435 L(shl_1_end):
436 lea 32(%ecx), %ecx
437 add %ecx, %edi
438 add %edi, %edx
439 lea 1(%edi, %eax), %eax
440 POP (%edi)
441 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
443 ALIGN (4)
444 L(shl_2):
445 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
446 lea -2(%eax), %eax
447 movaps (%eax), %xmm1
448 xor %edi, %edi
449 lea -32(%ecx), %ecx
450 movdqu %xmm0, (%esi)
451 POP (%esi)
452 L(shl_2_loop):
454 movdqa 16(%eax, %edi), %xmm2
455 sub $32, %ecx
456 movdqa 32(%eax, %edi), %xmm3
457 movdqa %xmm3, %xmm4
458 palignr $2, %xmm2, %xmm3
459 palignr $2, %xmm1, %xmm2
460 lea 32(%edi), %edi
461 movdqa %xmm2, -32(%edx, %edi)
462 movdqa %xmm3, -16(%edx, %edi)
464 jl L(shl_2_end)
466 movdqa 16(%eax, %edi), %xmm2
467 sub $32, %ecx
468 movdqa 32(%eax, %edi), %xmm3
469 movdqa %xmm3, %xmm1
470 palignr $2, %xmm2, %xmm3
471 palignr $2, %xmm4, %xmm2
472 lea 32(%edi), %edi
473 movdqa %xmm2, -32(%edx, %edi)
474 movdqa %xmm3, -16(%edx, %edi)
476 jae L(shl_2_loop)
478 L(shl_2_end):
479 lea 32(%ecx), %ecx
480 add %ecx, %edi
481 add %edi, %edx
482 lea 2(%edi, %eax), %eax
483 POP (%edi)
484 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
486 ALIGN (4)
487 L(shl_3):
488 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
489 lea -3(%eax), %eax
490 movaps (%eax), %xmm1
491 xor %edi, %edi
492 lea -32(%ecx), %ecx
493 movdqu %xmm0, (%esi)
494 POP (%esi)
495 L(shl_3_loop):
497 movdqa 16(%eax, %edi), %xmm2
498 sub $32, %ecx
499 movdqa 32(%eax, %edi), %xmm3
500 movdqa %xmm3, %xmm4
501 palignr $3, %xmm2, %xmm3
502 palignr $3, %xmm1, %xmm2
503 lea 32(%edi), %edi
504 movdqa %xmm2, -32(%edx, %edi)
505 movdqa %xmm3, -16(%edx, %edi)
507 jl L(shl_3_end)
509 movdqa 16(%eax, %edi), %xmm2
510 sub $32, %ecx
511 movdqa 32(%eax, %edi), %xmm3
512 movdqa %xmm3, %xmm1
513 palignr $3, %xmm2, %xmm3
514 palignr $3, %xmm4, %xmm2
515 lea 32(%edi), %edi
516 movdqa %xmm2, -32(%edx, %edi)
517 movdqa %xmm3, -16(%edx, %edi)
519 jae L(shl_3_loop)
521 L(shl_3_end):
522 lea 32(%ecx), %ecx
523 add %ecx, %edi
524 add %edi, %edx
525 lea 3(%edi, %eax), %eax
526 POP (%edi)
527 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
529 ALIGN (4)
530 L(shl_4):
531 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
532 lea -4(%eax), %eax
533 movaps (%eax), %xmm1
534 xor %edi, %edi
535 lea -32(%ecx), %ecx
536 movdqu %xmm0, (%esi)
537 POP (%esi)
538 L(shl_4_loop):
540 movdqa 16(%eax, %edi), %xmm2
541 sub $32, %ecx
542 movdqa 32(%eax, %edi), %xmm3
543 movdqa %xmm3, %xmm4
544 palignr $4, %xmm2, %xmm3
545 palignr $4, %xmm1, %xmm2
546 lea 32(%edi), %edi
547 movdqa %xmm2, -32(%edx, %edi)
548 movdqa %xmm3, -16(%edx, %edi)
550 jl L(shl_4_end)
552 movdqa 16(%eax, %edi), %xmm2
553 sub $32, %ecx
554 movdqa 32(%eax, %edi), %xmm3
555 movdqa %xmm3, %xmm1
556 palignr $4, %xmm2, %xmm3
557 palignr $4, %xmm4, %xmm2
558 lea 32(%edi), %edi
559 movdqa %xmm2, -32(%edx, %edi)
560 movdqa %xmm3, -16(%edx, %edi)
562 jae L(shl_4_loop)
564 L(shl_4_end):
565 lea 32(%ecx), %ecx
566 add %ecx, %edi
567 add %edi, %edx
568 lea 4(%edi, %eax), %eax
569 POP (%edi)
570 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
572 ALIGN (4)
573 L(shl_5):
574 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
575 lea -5(%eax), %eax
576 movaps (%eax), %xmm1
577 xor %edi, %edi
578 lea -32(%ecx), %ecx
579 movdqu %xmm0, (%esi)
580 POP (%esi)
581 L(shl_5_loop):
583 movdqa 16(%eax, %edi), %xmm2
584 sub $32, %ecx
585 movdqa 32(%eax, %edi), %xmm3
586 movdqa %xmm3, %xmm4
587 palignr $5, %xmm2, %xmm3
588 palignr $5, %xmm1, %xmm2
589 lea 32(%edi), %edi
590 movdqa %xmm2, -32(%edx, %edi)
591 movdqa %xmm3, -16(%edx, %edi)
593 jl L(shl_5_end)
595 movdqa 16(%eax, %edi), %xmm2
596 sub $32, %ecx
597 movdqa 32(%eax, %edi), %xmm3
598 movdqa %xmm3, %xmm1
599 palignr $5, %xmm2, %xmm3
600 palignr $5, %xmm4, %xmm2
601 lea 32(%edi), %edi
602 movdqa %xmm2, -32(%edx, %edi)
603 movdqa %xmm3, -16(%edx, %edi)
605 jae L(shl_5_loop)
607 L(shl_5_end):
608 lea 32(%ecx), %ecx
609 add %ecx, %edi
610 add %edi, %edx
611 lea 5(%edi, %eax), %eax
612 POP (%edi)
613 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
616 ALIGN (4)
617 L(shl_6):
618 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
619 lea -6(%eax), %eax
620 movaps (%eax), %xmm1
621 xor %edi, %edi
622 lea -32(%ecx), %ecx
623 movdqu %xmm0, (%esi)
624 POP (%esi)
625 L(shl_6_loop):
627 movdqa 16(%eax, %edi), %xmm2
628 sub $32, %ecx
629 movdqa 32(%eax, %edi), %xmm3
630 movdqa %xmm3, %xmm4
631 palignr $6, %xmm2, %xmm3
632 palignr $6, %xmm1, %xmm2
633 lea 32(%edi), %edi
634 movdqa %xmm2, -32(%edx, %edi)
635 movdqa %xmm3, -16(%edx, %edi)
637 jl L(shl_6_end)
639 movdqa 16(%eax, %edi), %xmm2
640 sub $32, %ecx
641 movdqa 32(%eax, %edi), %xmm3
642 movdqa %xmm3, %xmm1
643 palignr $6, %xmm2, %xmm3
644 palignr $6, %xmm4, %xmm2
645 lea 32(%edi), %edi
646 movdqa %xmm2, -32(%edx, %edi)
647 movdqa %xmm3, -16(%edx, %edi)
649 jae L(shl_6_loop)
651 L(shl_6_end):
652 lea 32(%ecx), %ecx
653 add %ecx, %edi
654 add %edi, %edx
655 lea 6(%edi, %eax), %eax
656 POP (%edi)
657 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
659 ALIGN (4)
660 L(shl_7):
661 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
662 lea -7(%eax), %eax
663 movaps (%eax), %xmm1
664 xor %edi, %edi
665 lea -32(%ecx), %ecx
666 movdqu %xmm0, (%esi)
667 POP (%esi)
668 L(shl_7_loop):
670 movdqa 16(%eax, %edi), %xmm2
671 sub $32, %ecx
672 movdqa 32(%eax, %edi), %xmm3
673 movdqa %xmm3, %xmm4
674 palignr $7, %xmm2, %xmm3
675 palignr $7, %xmm1, %xmm2
676 lea 32(%edi), %edi
677 movdqa %xmm2, -32(%edx, %edi)
678 movdqa %xmm3, -16(%edx, %edi)
680 jl L(shl_7_end)
682 movdqa 16(%eax, %edi), %xmm2
683 sub $32, %ecx
684 movdqa 32(%eax, %edi), %xmm3
685 movdqa %xmm3, %xmm1
686 palignr $7, %xmm2, %xmm3
687 palignr $7, %xmm4, %xmm2
688 lea 32(%edi), %edi
689 movdqa %xmm2, -32(%edx, %edi)
690 movdqa %xmm3, -16(%edx, %edi)
692 jae L(shl_7_loop)
694 L(shl_7_end):
695 lea 32(%ecx), %ecx
696 add %ecx, %edi
697 add %edi, %edx
698 lea 7(%edi, %eax), %eax
699 POP (%edi)
700 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
702 ALIGN (4)
703 L(shl_8):
704 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
705 lea -8(%eax), %eax
706 movaps (%eax), %xmm1
707 xor %edi, %edi
708 lea -32(%ecx), %ecx
709 movdqu %xmm0, (%esi)
710 POP (%esi)
711 L(shl_8_loop):
713 movdqa 16(%eax, %edi), %xmm2
714 sub $32, %ecx
715 movdqa 32(%eax, %edi), %xmm3
716 movdqa %xmm3, %xmm4
717 palignr $8, %xmm2, %xmm3
718 palignr $8, %xmm1, %xmm2
719 lea 32(%edi), %edi
720 movdqa %xmm2, -32(%edx, %edi)
721 movdqa %xmm3, -16(%edx, %edi)
723 jl L(shl_8_end)
725 movdqa 16(%eax, %edi), %xmm2
726 sub $32, %ecx
727 movdqa 32(%eax, %edi), %xmm3
728 movdqa %xmm3, %xmm1
729 palignr $8, %xmm2, %xmm3
730 palignr $8, %xmm4, %xmm2
731 lea 32(%edi), %edi
732 movdqa %xmm2, -32(%edx, %edi)
733 movdqa %xmm3, -16(%edx, %edi)
735 jae L(shl_8_loop)
737 L(shl_8_end):
738 lea 32(%ecx), %ecx
739 add %ecx, %edi
740 add %edi, %edx
741 lea 8(%edi, %eax), %eax
742 POP (%edi)
743 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
745 ALIGN (4)
746 L(shl_9):
747 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
748 lea -9(%eax), %eax
749 movaps (%eax), %xmm1
750 xor %edi, %edi
751 lea -32(%ecx), %ecx
752 movdqu %xmm0, (%esi)
753 POP (%esi)
754 L(shl_9_loop):
756 movdqa 16(%eax, %edi), %xmm2
757 sub $32, %ecx
758 movdqa 32(%eax, %edi), %xmm3
759 movdqa %xmm3, %xmm4
760 palignr $9, %xmm2, %xmm3
761 palignr $9, %xmm1, %xmm2
762 lea 32(%edi), %edi
763 movdqa %xmm2, -32(%edx, %edi)
764 movdqa %xmm3, -16(%edx, %edi)
766 jl L(shl_9_end)
768 movdqa 16(%eax, %edi), %xmm2
769 sub $32, %ecx
770 movdqa 32(%eax, %edi), %xmm3
771 movdqa %xmm3, %xmm1
772 palignr $9, %xmm2, %xmm3
773 palignr $9, %xmm4, %xmm2
774 lea 32(%edi), %edi
775 movdqa %xmm2, -32(%edx, %edi)
776 movdqa %xmm3, -16(%edx, %edi)
778 jae L(shl_9_loop)
780 L(shl_9_end):
781 lea 32(%ecx), %ecx
782 add %ecx, %edi
783 add %edi, %edx
784 lea 9(%edi, %eax), %eax
785 POP (%edi)
786 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
788 ALIGN (4)
789 L(shl_10):
790 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
791 lea -10(%eax), %eax
792 movaps (%eax), %xmm1
793 xor %edi, %edi
794 lea -32(%ecx), %ecx
795 movdqu %xmm0, (%esi)
796 POP (%esi)
797 L(shl_10_loop):
799 movdqa 16(%eax, %edi), %xmm2
800 sub $32, %ecx
801 movdqa 32(%eax, %edi), %xmm3
802 movdqa %xmm3, %xmm4
803 palignr $10, %xmm2, %xmm3
804 palignr $10, %xmm1, %xmm2
805 lea 32(%edi), %edi
806 movdqa %xmm2, -32(%edx, %edi)
807 movdqa %xmm3, -16(%edx, %edi)
809 jl L(shl_10_end)
811 movdqa 16(%eax, %edi), %xmm2
812 sub $32, %ecx
813 movdqa 32(%eax, %edi), %xmm3
814 movdqa %xmm3, %xmm1
815 palignr $10, %xmm2, %xmm3
816 palignr $10, %xmm4, %xmm2
817 lea 32(%edi), %edi
818 movdqa %xmm2, -32(%edx, %edi)
819 movdqa %xmm3, -16(%edx, %edi)
821 jae L(shl_10_loop)
823 L(shl_10_end):
824 lea 32(%ecx), %ecx
825 add %ecx, %edi
826 add %edi, %edx
827 lea 10(%edi, %eax), %eax
828 POP (%edi)
829 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
831 ALIGN (4)
832 L(shl_11):
833 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
834 lea -11(%eax), %eax
835 movaps (%eax), %xmm1
836 xor %edi, %edi
837 lea -32(%ecx), %ecx
838 movdqu %xmm0, (%esi)
839 POP (%esi)
840 L(shl_11_loop):
842 movdqa 16(%eax, %edi), %xmm2
843 sub $32, %ecx
844 movdqa 32(%eax, %edi), %xmm3
845 movdqa %xmm3, %xmm4
846 palignr $11, %xmm2, %xmm3
847 palignr $11, %xmm1, %xmm2
848 lea 32(%edi), %edi
849 movdqa %xmm2, -32(%edx, %edi)
850 movdqa %xmm3, -16(%edx, %edi)
852 jl L(shl_11_end)
854 movdqa 16(%eax, %edi), %xmm2
855 sub $32, %ecx
856 movdqa 32(%eax, %edi), %xmm3
857 movdqa %xmm3, %xmm1
858 palignr $11, %xmm2, %xmm3
859 palignr $11, %xmm4, %xmm2
860 lea 32(%edi), %edi
861 movdqa %xmm2, -32(%edx, %edi)
862 movdqa %xmm3, -16(%edx, %edi)
864 jae L(shl_11_loop)
866 L(shl_11_end):
867 lea 32(%ecx), %ecx
868 add %ecx, %edi
869 add %edi, %edx
870 lea 11(%edi, %eax), %eax
871 POP (%edi)
872 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
874 ALIGN (4)
875 L(shl_12):
876 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
877 lea -12(%eax), %eax
878 movaps (%eax), %xmm1
879 xor %edi, %edi
880 lea -32(%ecx), %ecx
881 movdqu %xmm0, (%esi)
882 POP (%esi)
883 L(shl_12_loop):
885 movdqa 16(%eax, %edi), %xmm2
886 sub $32, %ecx
887 movdqa 32(%eax, %edi), %xmm3
888 movdqa %xmm3, %xmm4
889 palignr $12, %xmm2, %xmm3
890 palignr $12, %xmm1, %xmm2
891 lea 32(%edi), %edi
892 movdqa %xmm2, -32(%edx, %edi)
893 movdqa %xmm3, -16(%edx, %edi)
895 jl L(shl_12_end)
897 movdqa 16(%eax, %edi), %xmm2
898 sub $32, %ecx
899 movdqa 32(%eax, %edi), %xmm3
900 movdqa %xmm3, %xmm1
901 palignr $12, %xmm2, %xmm3
902 palignr $12, %xmm4, %xmm2
903 lea 32(%edi), %edi
904 movdqa %xmm2, -32(%edx, %edi)
905 movdqa %xmm3, -16(%edx, %edi)
907 jae L(shl_12_loop)
909 L(shl_12_end):
910 lea 32(%ecx), %ecx
911 add %ecx, %edi
912 add %edi, %edx
913 lea 12(%edi, %eax), %eax
914 POP (%edi)
915 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
917 ALIGN (4)
918 L(shl_13):
919 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
920 lea -13(%eax), %eax
921 movaps (%eax), %xmm1
922 xor %edi, %edi
923 lea -32(%ecx), %ecx
924 movdqu %xmm0, (%esi)
925 POP (%esi)
926 L(shl_13_loop):
928 movdqa 16(%eax, %edi), %xmm2
929 sub $32, %ecx
930 movdqa 32(%eax, %edi), %xmm3
931 movdqa %xmm3, %xmm4
932 palignr $13, %xmm2, %xmm3
933 palignr $13, %xmm1, %xmm2
934 lea 32(%edi), %edi
935 movdqa %xmm2, -32(%edx, %edi)
936 movdqa %xmm3, -16(%edx, %edi)
938 jl L(shl_13_end)
940 movdqa 16(%eax, %edi), %xmm2
941 sub $32, %ecx
942 movdqa 32(%eax, %edi), %xmm3
943 movdqa %xmm3, %xmm1
944 palignr $13, %xmm2, %xmm3
945 palignr $13, %xmm4, %xmm2
946 lea 32(%edi), %edi
947 movdqa %xmm2, -32(%edx, %edi)
948 movdqa %xmm3, -16(%edx, %edi)
950 jae L(shl_13_loop)
952 L(shl_13_end):
953 lea 32(%ecx), %ecx
954 add %ecx, %edi
955 add %edi, %edx
956 lea 13(%edi, %eax), %eax
957 POP (%edi)
958 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
960 ALIGN (4)
961 L(shl_14):
962 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
963 lea -14(%eax), %eax
964 movaps (%eax), %xmm1
965 xor %edi, %edi
966 lea -32(%ecx), %ecx
967 movdqu %xmm0, (%esi)
968 POP (%esi)
969 L(shl_14_loop):
971 movdqa 16(%eax, %edi), %xmm2
972 sub $32, %ecx
973 movdqa 32(%eax, %edi), %xmm3
974 movdqa %xmm3, %xmm4
975 palignr $14, %xmm2, %xmm3
976 palignr $14, %xmm1, %xmm2
977 lea 32(%edi), %edi
978 movdqa %xmm2, -32(%edx, %edi)
979 movdqa %xmm3, -16(%edx, %edi)
981 jl L(shl_14_end)
983 movdqa 16(%eax, %edi), %xmm2
984 sub $32, %ecx
985 movdqa 32(%eax, %edi), %xmm3
986 movdqa %xmm3, %xmm1
987 palignr $14, %xmm2, %xmm3
988 palignr $14, %xmm4, %xmm2
989 lea 32(%edi), %edi
990 movdqa %xmm2, -32(%edx, %edi)
991 movdqa %xmm3, -16(%edx, %edi)
993 jae L(shl_14_loop)
995 L(shl_14_end):
996 lea 32(%ecx), %ecx
997 add %ecx, %edi
998 add %edi, %edx
999 lea 14(%edi, %eax), %eax
1000 POP (%edi)
1001 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
1004 ALIGN (4)
1005 L(shl_15):
1006 BRANCH_TO_JMPTBL_ENTRY_VALUE(L(table_48bytes_fwd))
1007 lea -15(%eax), %eax
1008 movaps (%eax), %xmm1
1009 xor %edi, %edi
1010 lea -32(%ecx), %ecx
1011 movdqu %xmm0, (%esi)
1012 POP (%esi)
1013 L(shl_15_loop):
1015 movdqa 16(%eax, %edi), %xmm2
1016 sub $32, %ecx
1017 movdqa 32(%eax, %edi), %xmm3
1018 movdqa %xmm3, %xmm4
1019 palignr $15, %xmm2, %xmm3
1020 palignr $15, %xmm1, %xmm2
1021 lea 32(%edi), %edi
1022 movdqa %xmm2, -32(%edx, %edi)
1023 movdqa %xmm3, -16(%edx, %edi)
1025 jl L(shl_15_end)
1027 movdqa 16(%eax, %edi), %xmm2
1028 sub $32, %ecx
1029 movdqa 32(%eax, %edi), %xmm3
1030 movdqa %xmm3, %xmm1
1031 palignr $15, %xmm2, %xmm3
1032 palignr $15, %xmm4, %xmm2
1033 lea 32(%edi), %edi
1034 movdqa %xmm2, -32(%edx, %edi)
1035 movdqa %xmm3, -16(%edx, %edi)
1037 jae L(shl_15_loop)
1039 L(shl_15_end):
1040 lea 32(%ecx), %ecx
1041 add %ecx, %edi
1042 add %edi, %edx
1043 lea 15(%edi, %eax), %eax
1044 POP (%edi)
1045 BRANCH_TO_JMPTBL_ENTRY_TAIL(L(table_48bytes_fwd), %ecx, 4)
1048 ALIGN (4)
1049 L(fwd_write_44bytes):
1050 movl -44(%eax), %ecx
1051 movl %ecx, -44(%edx)
1052 L(fwd_write_40bytes):
1053 movl -40(%eax), %ecx
1054 movl %ecx, -40(%edx)
1055 L(fwd_write_36bytes):
1056 movl -36(%eax), %ecx
1057 movl %ecx, -36(%edx)
1058 L(fwd_write_32bytes):
1059 movl -32(%eax), %ecx
1060 movl %ecx, -32(%edx)
1061 L(fwd_write_28bytes):
1062 movl -28(%eax), %ecx
1063 movl %ecx, -28(%edx)
1064 L(fwd_write_24bytes):
1065 movl -24(%eax), %ecx
1066 movl %ecx, -24(%edx)
1067 L(fwd_write_20bytes):
1068 movl -20(%eax), %ecx
1069 movl %ecx, -20(%edx)
1070 L(fwd_write_16bytes):
1071 movl -16(%eax), %ecx
1072 movl %ecx, -16(%edx)
1073 L(fwd_write_12bytes):
1074 movl -12(%eax), %ecx
1075 movl %ecx, -12(%edx)
1076 L(fwd_write_8bytes):
1077 movl -8(%eax), %ecx
1078 movl %ecx, -8(%edx)
1079 L(fwd_write_4bytes):
1080 movl -4(%eax), %ecx
1081 movl %ecx, -4(%edx)
1082 L(fwd_write_0bytes):
1083 #ifndef USE_AS_BCOPY
1084 # ifdef USE_AS_MEMPCPY
1085 movl %edx, %eax
1086 # else
1087 movl DEST(%esp), %eax
1088 # endif
1089 #endif
1090 RETURN
1092 ALIGN (4)
1093 L(fwd_write_5bytes):
1094 movl -5(%eax), %ecx
1095 movl -4(%eax), %eax
1096 movl %ecx, -5(%edx)
1097 movl %eax, -4(%edx)
1098 #ifndef USE_AS_BCOPY
1099 # ifdef USE_AS_MEMPCPY
1100 movl %edx, %eax
1101 # else
1102 movl DEST(%esp), %eax
1103 # endif
1104 #endif
1105 RETURN
1107 ALIGN (4)
1108 L(fwd_write_45bytes):
1109 movl -45(%eax), %ecx
1110 movl %ecx, -45(%edx)
1111 L(fwd_write_41bytes):
1112 movl -41(%eax), %ecx
1113 movl %ecx, -41(%edx)
1114 L(fwd_write_37bytes):
1115 movl -37(%eax), %ecx
1116 movl %ecx, -37(%edx)
1117 L(fwd_write_33bytes):
1118 movl -33(%eax), %ecx
1119 movl %ecx, -33(%edx)
1120 L(fwd_write_29bytes):
1121 movl -29(%eax), %ecx
1122 movl %ecx, -29(%edx)
1123 L(fwd_write_25bytes):
1124 movl -25(%eax), %ecx
1125 movl %ecx, -25(%edx)
1126 L(fwd_write_21bytes):
1127 movl -21(%eax), %ecx
1128 movl %ecx, -21(%edx)
1129 L(fwd_write_17bytes):
1130 movl -17(%eax), %ecx
1131 movl %ecx, -17(%edx)
1132 L(fwd_write_13bytes):
1133 movl -13(%eax), %ecx
1134 movl %ecx, -13(%edx)
1135 L(fwd_write_9bytes):
1136 movl -9(%eax), %ecx
1137 movl %ecx, -9(%edx)
1138 movl -5(%eax), %ecx
1139 movl %ecx, -5(%edx)
1140 L(fwd_write_1bytes):
1141 movzbl -1(%eax), %ecx
1142 movb %cl, -1(%edx)
1143 #ifndef USE_AS_BCOPY
1144 # ifdef USE_AS_MEMPCPY
1145 movl %edx, %eax
1146 # else
1147 movl DEST(%esp), %eax
1148 # endif
1149 #endif
1150 RETURN
1152 ALIGN (4)
1153 L(fwd_write_46bytes):
1154 movl -46(%eax), %ecx
1155 movl %ecx, -46(%edx)
1156 L(fwd_write_42bytes):
1157 movl -42(%eax), %ecx
1158 movl %ecx, -42(%edx)
1159 L(fwd_write_38bytes):
1160 movl -38(%eax), %ecx
1161 movl %ecx, -38(%edx)
1162 L(fwd_write_34bytes):
1163 movl -34(%eax), %ecx
1164 movl %ecx, -34(%edx)
1165 L(fwd_write_30bytes):
1166 movl -30(%eax), %ecx
1167 movl %ecx, -30(%edx)
1168 L(fwd_write_26bytes):
1169 movl -26(%eax), %ecx
1170 movl %ecx, -26(%edx)
1171 L(fwd_write_22bytes):
1172 movl -22(%eax), %ecx
1173 movl %ecx, -22(%edx)
1174 L(fwd_write_18bytes):
1175 movl -18(%eax), %ecx
1176 movl %ecx, -18(%edx)
1177 L(fwd_write_14bytes):
1178 movl -14(%eax), %ecx
1179 movl %ecx, -14(%edx)
1180 L(fwd_write_10bytes):
1181 movl -10(%eax), %ecx
1182 movl %ecx, -10(%edx)
1183 L(fwd_write_6bytes):
1184 movl -6(%eax), %ecx
1185 movl %ecx, -6(%edx)
1186 L(fwd_write_2bytes):
1187 movzwl -2(%eax), %ecx
1188 movw %cx, -2(%edx)
1189 #ifndef USE_AS_BCOPY
1190 # ifdef USE_AS_MEMPCPY
1191 movl %edx, %eax
1192 # else
1193 movl DEST(%esp), %eax
1194 # endif
1195 #endif
1196 RETURN
1198 ALIGN (4)
1199 L(fwd_write_47bytes):
1200 movl -47(%eax), %ecx
1201 movl %ecx, -47(%edx)
1202 L(fwd_write_43bytes):
1203 movl -43(%eax), %ecx
1204 movl %ecx, -43(%edx)
1205 L(fwd_write_39bytes):
1206 movl -39(%eax), %ecx
1207 movl %ecx, -39(%edx)
1208 L(fwd_write_35bytes):
1209 movl -35(%eax), %ecx
1210 movl %ecx, -35(%edx)
1211 L(fwd_write_31bytes):
1212 movl -31(%eax), %ecx
1213 movl %ecx, -31(%edx)
1214 L(fwd_write_27bytes):
1215 movl -27(%eax), %ecx
1216 movl %ecx, -27(%edx)
1217 L(fwd_write_23bytes):
1218 movl -23(%eax), %ecx
1219 movl %ecx, -23(%edx)
1220 L(fwd_write_19bytes):
1221 movl -19(%eax), %ecx
1222 movl %ecx, -19(%edx)
1223 L(fwd_write_15bytes):
1224 movl -15(%eax), %ecx
1225 movl %ecx, -15(%edx)
1226 L(fwd_write_11bytes):
1227 movl -11(%eax), %ecx
1228 movl %ecx, -11(%edx)
1229 L(fwd_write_7bytes):
1230 movl -7(%eax), %ecx
1231 movl %ecx, -7(%edx)
1232 L(fwd_write_3bytes):
1233 movzwl -3(%eax), %ecx
1234 movzbl -1(%eax), %eax
1235 movw %cx, -3(%edx)
1236 movb %al, -1(%edx)
1237 #ifndef USE_AS_BCOPY
1238 # ifdef USE_AS_MEMPCPY
1239 movl %edx, %eax
1240 # else
1241 movl DEST(%esp), %eax
1242 # endif
1243 #endif
1244 RETURN
1246 ALIGN (4)
1247 L(large_page):
1248 movdqu (%eax), %xmm1
1249 lea 16(%eax), %eax
1250 movdqu %xmm0, (%esi)
1251 movntdq %xmm1, (%edx)
1252 lea 16(%edx), %edx
1253 cmp %al, %dl
1254 je L(copy_page_by_rep)
1255 L(large_page_loop_init):
1256 POP (%esi)
1257 lea -0x90(%ecx), %ecx
1258 POP (%edi)
1259 L(large_page_loop):
1260 prefetchnta 0x1c0(%eax)
1261 prefetchnta 0x280(%eax)
1262 movdqu (%eax), %xmm0
1263 movdqu 0x10(%eax), %xmm1
1264 movdqu 0x20(%eax), %xmm2
1265 movdqu 0x30(%eax), %xmm3
1266 movdqu 0x40(%eax), %xmm4
1267 movdqu 0x50(%eax), %xmm5
1268 movdqu 0x60(%eax), %xmm6
1269 movdqu 0x70(%eax), %xmm7
1270 lea 0x80(%eax), %eax
1271 lfence
1272 sub $0x80, %ecx
1273 movntdq %xmm0, (%edx)
1274 movntdq %xmm1, 0x10(%edx)
1275 movntdq %xmm2, 0x20(%edx)
1276 movntdq %xmm3, 0x30(%edx)
1277 movntdq %xmm4, 0x40(%edx)
1278 movntdq %xmm5, 0x50(%edx)
1279 movntdq %xmm6, 0x60(%edx)
1280 movntdq %xmm7, 0x70(%edx)
1281 lea 0x80(%edx), %edx
1282 jae L(large_page_loop)
1283 cmp $-0x40, %ecx
1284 lea 0x80(%ecx), %ecx
1285 jl L(large_page_less_64bytes)
1287 movdqu (%eax), %xmm0
1288 movdqu 0x10(%eax), %xmm1
1289 movdqu 0x20(%eax), %xmm2
1290 movdqu 0x30(%eax), %xmm3
1291 lea 0x40(%eax), %eax
1293 movntdq %xmm0, (%edx)
1294 movntdq %xmm1, 0x10(%edx)
1295 movntdq %xmm2, 0x20(%edx)
1296 movntdq %xmm3, 0x30(%edx)
1297 lea 0x40(%edx), %edx
1298 sub $0x40, %ecx
1299 L(large_page_less_64bytes):
1300 cmp $32, %ecx
1301 jl L(large_page_less_32bytes)
1302 movdqu (%eax), %xmm0
1303 movdqu 0x10(%eax), %xmm1
1304 lea 0x20(%eax), %eax
1305 movntdq %xmm0, (%edx)
1306 movntdq %xmm1, 0x10(%edx)
1307 lea 0x20(%edx), %edx
1308 sub $0x20, %ecx
1309 L(large_page_less_32bytes):
1310 add %ecx, %edx
1311 add %ecx, %eax
1312 sfence
1313 BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
1315 ALIGN (4)
1316 L(copy_page_by_rep):
1317 mov %eax, %esi
1318 mov %edx, %edi
1319 mov %ecx, %edx
1320 shr $2, %ecx
1321 and $3, %edx
1322 rep movsl
1323 jz L(copy_page_by_rep_exit)
1324 cmp $2, %edx
1325 jb L(copy_page_by_rep_left_1)
1326 movzwl (%esi), %eax
1327 movw %ax, (%edi)
1328 add $2, %esi
1329 add $2, %edi
1330 sub $2, %edx
1331 jz L(copy_page_by_rep_exit)
1332 L(copy_page_by_rep_left_1):
1333 movzbl (%esi), %eax
1334 movb %al, (%edi)
1335 L(copy_page_by_rep_exit):
1336 POP (%esi)
1337 POP (%edi)
1338 #ifndef USE_AS_BCOPY
1339 movl DEST(%esp), %eax
1340 # ifdef USE_AS_MEMPCPY
1341 movl LEN(%esp), %ecx
1342 add %ecx, %eax
1343 # endif
1344 #endif
1345 RETURN
1347 ALIGN (4)
1348 L(bk_write_44bytes):
1349 movl 40(%eax), %ecx
1350 movl %ecx, 40(%edx)
1351 L(bk_write_40bytes):
1352 movl 36(%eax), %ecx
1353 movl %ecx, 36(%edx)
1354 L(bk_write_36bytes):
1355 movl 32(%eax), %ecx
1356 movl %ecx, 32(%edx)
1357 L(bk_write_32bytes):
1358 movl 28(%eax), %ecx
1359 movl %ecx, 28(%edx)
1360 L(bk_write_28bytes):
1361 movl 24(%eax), %ecx
1362 movl %ecx, 24(%edx)
1363 L(bk_write_24bytes):
1364 movl 20(%eax), %ecx
1365 movl %ecx, 20(%edx)
1366 L(bk_write_20bytes):
1367 movl 16(%eax), %ecx
1368 movl %ecx, 16(%edx)
1369 L(bk_write_16bytes):
1370 movl 12(%eax), %ecx
1371 movl %ecx, 12(%edx)
1372 L(bk_write_12bytes):
1373 movl 8(%eax), %ecx
1374 movl %ecx, 8(%edx)
1375 L(bk_write_8bytes):
1376 movl 4(%eax), %ecx
1377 movl %ecx, 4(%edx)
1378 L(bk_write_4bytes):
1379 movl (%eax), %ecx
1380 movl %ecx, (%edx)
1381 L(bk_write_0bytes):
1382 #ifndef USE_AS_BCOPY
1383 movl DEST(%esp), %eax
1384 # ifdef USE_AS_MEMPCPY
1385 movl LEN(%esp), %ecx
1386 add %ecx, %eax
1387 # endif
1388 #endif
1389 RETURN
1391 ALIGN (4)
1392 L(bk_write_45bytes):
1393 movl 41(%eax), %ecx
1394 movl %ecx, 41(%edx)
1395 L(bk_write_41bytes):
1396 movl 37(%eax), %ecx
1397 movl %ecx, 37(%edx)
1398 L(bk_write_37bytes):
1399 movl 33(%eax), %ecx
1400 movl %ecx, 33(%edx)
1401 L(bk_write_33bytes):
1402 movl 29(%eax), %ecx
1403 movl %ecx, 29(%edx)
1404 L(bk_write_29bytes):
1405 movl 25(%eax), %ecx
1406 movl %ecx, 25(%edx)
1407 L(bk_write_25bytes):
1408 movl 21(%eax), %ecx
1409 movl %ecx, 21(%edx)
1410 L(bk_write_21bytes):
1411 movl 17(%eax), %ecx
1412 movl %ecx, 17(%edx)
1413 L(bk_write_17bytes):
1414 movl 13(%eax), %ecx
1415 movl %ecx, 13(%edx)
1416 L(bk_write_13bytes):
1417 movl 9(%eax), %ecx
1418 movl %ecx, 9(%edx)
1419 L(bk_write_9bytes):
1420 movl 5(%eax), %ecx
1421 movl %ecx, 5(%edx)
1422 L(bk_write_5bytes):
1423 movl 1(%eax), %ecx
1424 movl %ecx, 1(%edx)
1425 L(bk_write_1bytes):
1426 movzbl (%eax), %ecx
1427 movb %cl, (%edx)
1428 #ifndef USE_AS_BCOPY
1429 movl DEST(%esp), %eax
1430 # ifdef USE_AS_MEMPCPY
1431 movl LEN(%esp), %ecx
1432 add %ecx, %eax
1433 # endif
1434 #endif
1435 RETURN
1437 ALIGN (4)
1438 L(bk_write_46bytes):
1439 movl 42(%eax), %ecx
1440 movl %ecx, 42(%edx)
1441 L(bk_write_42bytes):
1442 movl 38(%eax), %ecx
1443 movl %ecx, 38(%edx)
1444 L(bk_write_38bytes):
1445 movl 34(%eax), %ecx
1446 movl %ecx, 34(%edx)
1447 L(bk_write_34bytes):
1448 movl 30(%eax), %ecx
1449 movl %ecx, 30(%edx)
1450 L(bk_write_30bytes):
1451 movl 26(%eax), %ecx
1452 movl %ecx, 26(%edx)
1453 L(bk_write_26bytes):
1454 movl 22(%eax), %ecx
1455 movl %ecx, 22(%edx)
1456 L(bk_write_22bytes):
1457 movl 18(%eax), %ecx
1458 movl %ecx, 18(%edx)
1459 L(bk_write_18bytes):
1460 movl 14(%eax), %ecx
1461 movl %ecx, 14(%edx)
1462 L(bk_write_14bytes):
1463 movl 10(%eax), %ecx
1464 movl %ecx, 10(%edx)
1465 L(bk_write_10bytes):
1466 movl 6(%eax), %ecx
1467 movl %ecx, 6(%edx)
1468 L(bk_write_6bytes):
1469 movl 2(%eax), %ecx
1470 movl %ecx, 2(%edx)
1471 L(bk_write_2bytes):
1472 movzwl (%eax), %ecx
1473 movw %cx, (%edx)
1474 #ifndef USE_AS_BCOPY
1475 movl DEST(%esp), %eax
1476 # ifdef USE_AS_MEMPCPY
1477 movl LEN(%esp), %ecx
1478 add %ecx, %eax
1479 # endif
1480 #endif
1481 RETURN
1483 ALIGN (4)
1484 L(bk_write_47bytes):
1485 movl 43(%eax), %ecx
1486 movl %ecx, 43(%edx)
1487 L(bk_write_43bytes):
1488 movl 39(%eax), %ecx
1489 movl %ecx, 39(%edx)
1490 L(bk_write_39bytes):
1491 movl 35(%eax), %ecx
1492 movl %ecx, 35(%edx)
1493 L(bk_write_35bytes):
1494 movl 31(%eax), %ecx
1495 movl %ecx, 31(%edx)
1496 L(bk_write_31bytes):
1497 movl 27(%eax), %ecx
1498 movl %ecx, 27(%edx)
1499 L(bk_write_27bytes):
1500 movl 23(%eax), %ecx
1501 movl %ecx, 23(%edx)
1502 L(bk_write_23bytes):
1503 movl 19(%eax), %ecx
1504 movl %ecx, 19(%edx)
1505 L(bk_write_19bytes):
1506 movl 15(%eax), %ecx
1507 movl %ecx, 15(%edx)
1508 L(bk_write_15bytes):
1509 movl 11(%eax), %ecx
1510 movl %ecx, 11(%edx)
1511 L(bk_write_11bytes):
1512 movl 7(%eax), %ecx
1513 movl %ecx, 7(%edx)
1514 L(bk_write_7bytes):
1515 movl 3(%eax), %ecx
1516 movl %ecx, 3(%edx)
1517 L(bk_write_3bytes):
1518 movzwl 1(%eax), %ecx
1519 movw %cx, 1(%edx)
1520 movzbl (%eax), %eax
1521 movb %al, (%edx)
1522 #ifndef USE_AS_BCOPY
1523 movl DEST(%esp), %eax
1524 # ifdef USE_AS_MEMPCPY
1525 movl LEN(%esp), %ecx
1526 add %ecx, %eax
1527 # endif
1528 #endif
1529 RETURN_END
1532 .pushsection .rodata.ssse3,"a",@progbits
1533 ALIGN (2)
1534 L(table_48bytes_fwd):
1535 .int JMPTBL (L(fwd_write_0bytes), L(table_48bytes_fwd))
1536 .int JMPTBL (L(fwd_write_1bytes), L(table_48bytes_fwd))
1537 .int JMPTBL (L(fwd_write_2bytes), L(table_48bytes_fwd))
1538 .int JMPTBL (L(fwd_write_3bytes), L(table_48bytes_fwd))
1539 .int JMPTBL (L(fwd_write_4bytes), L(table_48bytes_fwd))
1540 .int JMPTBL (L(fwd_write_5bytes), L(table_48bytes_fwd))
1541 .int JMPTBL (L(fwd_write_6bytes), L(table_48bytes_fwd))
1542 .int JMPTBL (L(fwd_write_7bytes), L(table_48bytes_fwd))
1543 .int JMPTBL (L(fwd_write_8bytes), L(table_48bytes_fwd))
1544 .int JMPTBL (L(fwd_write_9bytes), L(table_48bytes_fwd))
1545 .int JMPTBL (L(fwd_write_10bytes), L(table_48bytes_fwd))
1546 .int JMPTBL (L(fwd_write_11bytes), L(table_48bytes_fwd))
1547 .int JMPTBL (L(fwd_write_12bytes), L(table_48bytes_fwd))
1548 .int JMPTBL (L(fwd_write_13bytes), L(table_48bytes_fwd))
1549 .int JMPTBL (L(fwd_write_14bytes), L(table_48bytes_fwd))
1550 .int JMPTBL (L(fwd_write_15bytes), L(table_48bytes_fwd))
1551 .int JMPTBL (L(fwd_write_16bytes), L(table_48bytes_fwd))
1552 .int JMPTBL (L(fwd_write_17bytes), L(table_48bytes_fwd))
1553 .int JMPTBL (L(fwd_write_18bytes), L(table_48bytes_fwd))
1554 .int JMPTBL (L(fwd_write_19bytes), L(table_48bytes_fwd))
1555 .int JMPTBL (L(fwd_write_20bytes), L(table_48bytes_fwd))
1556 .int JMPTBL (L(fwd_write_21bytes), L(table_48bytes_fwd))
1557 .int JMPTBL (L(fwd_write_22bytes), L(table_48bytes_fwd))
1558 .int JMPTBL (L(fwd_write_23bytes), L(table_48bytes_fwd))
1559 .int JMPTBL (L(fwd_write_24bytes), L(table_48bytes_fwd))
1560 .int JMPTBL (L(fwd_write_25bytes), L(table_48bytes_fwd))
1561 .int JMPTBL (L(fwd_write_26bytes), L(table_48bytes_fwd))
1562 .int JMPTBL (L(fwd_write_27bytes), L(table_48bytes_fwd))
1563 .int JMPTBL (L(fwd_write_28bytes), L(table_48bytes_fwd))
1564 .int JMPTBL (L(fwd_write_29bytes), L(table_48bytes_fwd))
1565 .int JMPTBL (L(fwd_write_30bytes), L(table_48bytes_fwd))
1566 .int JMPTBL (L(fwd_write_31bytes), L(table_48bytes_fwd))
1567 .int JMPTBL (L(fwd_write_32bytes), L(table_48bytes_fwd))
1568 .int JMPTBL (L(fwd_write_33bytes), L(table_48bytes_fwd))
1569 .int JMPTBL (L(fwd_write_34bytes), L(table_48bytes_fwd))
1570 .int JMPTBL (L(fwd_write_35bytes), L(table_48bytes_fwd))
1571 .int JMPTBL (L(fwd_write_36bytes), L(table_48bytes_fwd))
1572 .int JMPTBL (L(fwd_write_37bytes), L(table_48bytes_fwd))
1573 .int JMPTBL (L(fwd_write_38bytes), L(table_48bytes_fwd))
1574 .int JMPTBL (L(fwd_write_39bytes), L(table_48bytes_fwd))
1575 .int JMPTBL (L(fwd_write_40bytes), L(table_48bytes_fwd))
1576 .int JMPTBL (L(fwd_write_41bytes), L(table_48bytes_fwd))
1577 .int JMPTBL (L(fwd_write_42bytes), L(table_48bytes_fwd))
1578 .int JMPTBL (L(fwd_write_43bytes), L(table_48bytes_fwd))
1579 .int JMPTBL (L(fwd_write_44bytes), L(table_48bytes_fwd))
1580 .int JMPTBL (L(fwd_write_45bytes), L(table_48bytes_fwd))
1581 .int JMPTBL (L(fwd_write_46bytes), L(table_48bytes_fwd))
1582 .int JMPTBL (L(fwd_write_47bytes), L(table_48bytes_fwd))
1584 ALIGN (2)
1585 L(shl_table):
1586 .int JMPTBL (L(shl_0), L(shl_table))
1587 .int JMPTBL (L(shl_1), L(shl_table))
1588 .int JMPTBL (L(shl_2), L(shl_table))
1589 .int JMPTBL (L(shl_3), L(shl_table))
1590 .int JMPTBL (L(shl_4), L(shl_table))
1591 .int JMPTBL (L(shl_5), L(shl_table))
1592 .int JMPTBL (L(shl_6), L(shl_table))
1593 .int JMPTBL (L(shl_7), L(shl_table))
1594 .int JMPTBL (L(shl_8), L(shl_table))
1595 .int JMPTBL (L(shl_9), L(shl_table))
1596 .int JMPTBL (L(shl_10), L(shl_table))
1597 .int JMPTBL (L(shl_11), L(shl_table))
1598 .int JMPTBL (L(shl_12), L(shl_table))
1599 .int JMPTBL (L(shl_13), L(shl_table))
1600 .int JMPTBL (L(shl_14), L(shl_table))
1601 .int JMPTBL (L(shl_15), L(shl_table))
1603 ALIGN (2)
1604 L(table_48_bytes_bwd):
1605 .int JMPTBL (L(bk_write_0bytes), L(table_48_bytes_bwd))
1606 .int JMPTBL (L(bk_write_1bytes), L(table_48_bytes_bwd))
1607 .int JMPTBL (L(bk_write_2bytes), L(table_48_bytes_bwd))
1608 .int JMPTBL (L(bk_write_3bytes), L(table_48_bytes_bwd))
1609 .int JMPTBL (L(bk_write_4bytes), L(table_48_bytes_bwd))
1610 .int JMPTBL (L(bk_write_5bytes), L(table_48_bytes_bwd))
1611 .int JMPTBL (L(bk_write_6bytes), L(table_48_bytes_bwd))
1612 .int JMPTBL (L(bk_write_7bytes), L(table_48_bytes_bwd))
1613 .int JMPTBL (L(bk_write_8bytes), L(table_48_bytes_bwd))
1614 .int JMPTBL (L(bk_write_9bytes), L(table_48_bytes_bwd))
1615 .int JMPTBL (L(bk_write_10bytes), L(table_48_bytes_bwd))
1616 .int JMPTBL (L(bk_write_11bytes), L(table_48_bytes_bwd))
1617 .int JMPTBL (L(bk_write_12bytes), L(table_48_bytes_bwd))
1618 .int JMPTBL (L(bk_write_13bytes), L(table_48_bytes_bwd))
1619 .int JMPTBL (L(bk_write_14bytes), L(table_48_bytes_bwd))
1620 .int JMPTBL (L(bk_write_15bytes), L(table_48_bytes_bwd))
1621 .int JMPTBL (L(bk_write_16bytes), L(table_48_bytes_bwd))
1622 .int JMPTBL (L(bk_write_17bytes), L(table_48_bytes_bwd))
1623 .int JMPTBL (L(bk_write_18bytes), L(table_48_bytes_bwd))
1624 .int JMPTBL (L(bk_write_19bytes), L(table_48_bytes_bwd))
1625 .int JMPTBL (L(bk_write_20bytes), L(table_48_bytes_bwd))
1626 .int JMPTBL (L(bk_write_21bytes), L(table_48_bytes_bwd))
1627 .int JMPTBL (L(bk_write_22bytes), L(table_48_bytes_bwd))
1628 .int JMPTBL (L(bk_write_23bytes), L(table_48_bytes_bwd))
1629 .int JMPTBL (L(bk_write_24bytes), L(table_48_bytes_bwd))
1630 .int JMPTBL (L(bk_write_25bytes), L(table_48_bytes_bwd))
1631 .int JMPTBL (L(bk_write_26bytes), L(table_48_bytes_bwd))
1632 .int JMPTBL (L(bk_write_27bytes), L(table_48_bytes_bwd))
1633 .int JMPTBL (L(bk_write_28bytes), L(table_48_bytes_bwd))
1634 .int JMPTBL (L(bk_write_29bytes), L(table_48_bytes_bwd))
1635 .int JMPTBL (L(bk_write_30bytes), L(table_48_bytes_bwd))
1636 .int JMPTBL (L(bk_write_31bytes), L(table_48_bytes_bwd))
1637 .int JMPTBL (L(bk_write_32bytes), L(table_48_bytes_bwd))
1638 .int JMPTBL (L(bk_write_33bytes), L(table_48_bytes_bwd))
1639 .int JMPTBL (L(bk_write_34bytes), L(table_48_bytes_bwd))
1640 .int JMPTBL (L(bk_write_35bytes), L(table_48_bytes_bwd))
1641 .int JMPTBL (L(bk_write_36bytes), L(table_48_bytes_bwd))
1642 .int JMPTBL (L(bk_write_37bytes), L(table_48_bytes_bwd))
1643 .int JMPTBL (L(bk_write_38bytes), L(table_48_bytes_bwd))
1644 .int JMPTBL (L(bk_write_39bytes), L(table_48_bytes_bwd))
1645 .int JMPTBL (L(bk_write_40bytes), L(table_48_bytes_bwd))
1646 .int JMPTBL (L(bk_write_41bytes), L(table_48_bytes_bwd))
1647 .int JMPTBL (L(bk_write_42bytes), L(table_48_bytes_bwd))
1648 .int JMPTBL (L(bk_write_43bytes), L(table_48_bytes_bwd))
1649 .int JMPTBL (L(bk_write_44bytes), L(table_48_bytes_bwd))
1650 .int JMPTBL (L(bk_write_45bytes), L(table_48_bytes_bwd))
1651 .int JMPTBL (L(bk_write_46bytes), L(table_48_bytes_bwd))
1652 .int JMPTBL (L(bk_write_47bytes), L(table_48_bytes_bwd))
1654 .popsection
1656 #ifdef USE_AS_MEMMOVE
1657 ALIGN (4)
1658 L(copy_backward):
1659 PUSH (%esi)
1660 movl %eax, %esi
1661 lea (%ecx,%edx,1),%edx
1662 lea (%ecx,%esi,1),%esi
1663 testl $0x3, %edx
1664 jnz L(bk_align)
1666 L(bk_aligned_4):
1667 cmp $64, %ecx
1668 jge L(bk_write_more64bytes)
1670 L(bk_write_64bytesless):
1671 cmp $32, %ecx
1672 jl L(bk_write_less32bytes)
1674 L(bk_write_more32bytes):
1675 /* Copy 32 bytes at a time. */
1676 sub $32, %ecx
1677 movl -4(%esi), %eax
1678 movl %eax, -4(%edx)
1679 movl -8(%esi), %eax
1680 movl %eax, -8(%edx)
1681 movl -12(%esi), %eax
1682 movl %eax, -12(%edx)
1683 movl -16(%esi), %eax
1684 movl %eax, -16(%edx)
1685 movl -20(%esi), %eax
1686 movl %eax, -20(%edx)
1687 movl -24(%esi), %eax
1688 movl %eax, -24(%edx)
1689 movl -28(%esi), %eax
1690 movl %eax, -28(%edx)
1691 movl -32(%esi), %eax
1692 movl %eax, -32(%edx)
1693 sub $32, %edx
1694 sub $32, %esi
1696 L(bk_write_less32bytes):
1697 movl %esi, %eax
1698 sub %ecx, %edx
1699 sub %ecx, %eax
1700 POP (%esi)
1701 L(bk_write_less32bytes_2):
1702 BRANCH_TO_JMPTBL_ENTRY (L(table_48_bytes_bwd), %ecx, 4)
1704 ALIGN (4)
1705 L(bk_align):
1706 cmp $8, %ecx
1707 jle L(bk_write_less32bytes)
1708 testl $1, %edx
1709 /* We get here only if (EDX & 3 ) != 0 so if (EDX & 1) ==0,
1710 then (EDX & 2) must be != 0. */
1711 jz L(bk_got2)
1712 sub $1, %esi
1713 sub $1, %ecx
1714 sub $1, %edx
1715 movzbl (%esi), %eax
1716 movb %al, (%edx)
1718 testl $2, %edx
1719 jz L(bk_aligned_4)
1721 L(bk_got2):
1722 sub $2, %esi
1723 sub $2, %ecx
1724 sub $2, %edx
1725 movzwl (%esi), %eax
1726 movw %ax, (%edx)
1727 jmp L(bk_aligned_4)
1729 ALIGN (4)
1730 L(bk_write_more64bytes):
1731 /* Check alignment of last byte. */
1732 testl $15, %edx
1733 jz L(bk_ssse3_cpy_pre)
1735 /* EDX is aligned 4 bytes, but not 16 bytes. */
1736 L(bk_ssse3_align):
1737 sub $4, %esi
1738 sub $4, %ecx
1739 sub $4, %edx
1740 movl (%esi), %eax
1741 movl %eax, (%edx)
1743 testl $15, %edx
1744 jz L(bk_ssse3_cpy_pre)
1746 sub $4, %esi
1747 sub $4, %ecx
1748 sub $4, %edx
1749 movl (%esi), %eax
1750 movl %eax, (%edx)
1752 testl $15, %edx
1753 jz L(bk_ssse3_cpy_pre)
1755 sub $4, %esi
1756 sub $4, %ecx
1757 sub $4, %edx
1758 movl (%esi), %eax
1759 movl %eax, (%edx)
1761 L(bk_ssse3_cpy_pre):
1762 cmp $64, %ecx
1763 jl L(bk_write_more32bytes)
1765 L(bk_ssse3_cpy):
1766 sub $64, %esi
1767 sub $64, %ecx
1768 sub $64, %edx
1769 movdqu 0x30(%esi), %xmm3
1770 movdqa %xmm3, 0x30(%edx)
1771 movdqu 0x20(%esi), %xmm2
1772 movdqa %xmm2, 0x20(%edx)
1773 movdqu 0x10(%esi), %xmm1
1774 movdqa %xmm1, 0x10(%edx)
1775 movdqu (%esi), %xmm0
1776 movdqa %xmm0, (%edx)
1777 cmp $64, %ecx
1778 jge L(bk_ssse3_cpy)
1779 jmp L(bk_write_64bytesless)
1781 #endif
1783 END (MEMCPY)
1785 #endif