1 /* Optimized memcpy implementation for PowerPC A2.
2 Copyright (C) 2010-2020 Free Software Foundation, Inc.
3 Contributed by Michael Brutman <brutman@us.ibm.com>.
4 This file is part of the GNU C Library.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <https://www.gnu.org/licenses/>. */
21 #include <rtld-global-offsets.h>
23 #define PREFETCH_AHEAD 4 /* no cache lines SRC prefetching ahead */
24 #define ZERO_AHEAD 2 /* no cache lines DST zeroing ahead */
30 dcbt 0,r4 /* Prefetch ONE SRC cacheline */
31 cmplwi cr1,r5,16 /* is size < 16 ? */
32 mr r6,r3 /* Copy dest reg to r6; */
36 /* Big copy (16 bytes or more)
38 Figure out how far to the nearest quadword boundary, or if we are
41 r3 - return value (always)
42 r4 - current source addr
44 r6 - current dest addr
47 neg r8,r3 /* LS 4 bits = # bytes to 8-byte dest bdry */
48 clrlwi r8,r8,32-4 /* align to 16byte boundary */
49 sub r7,r4,r3 /* compute offset to src from dest */
50 cmplwi cr0,r8,0 /* Were we aligned on a 16 byte bdy? */
55 /* Destination is not aligned on quadword boundary. Get us to one.
57 r3 - return value (always)
58 r4 - current source addr
60 r6 - current dest addr
61 r7 - offset to src from dest
62 r8 - number of bytes to quadword boundary
65 mtcrf 0x01,r8 /* put #bytes to boundary into cr7 */
66 subf r5,r8,r5 /* adjust remaining len */
69 lbzx r0,r7,r6 /* copy 1 byte addr */
74 lhzx r0,r7,r6 /* copy 2 byte addr */
79 lwzx r0,r7,r6 /* copy 4 byte addr */
84 lfdx r0,r7,r6 /* copy 8 byte addr */
88 add r4,r7,r6 /* update src addr */
92 /* Dest is quadword aligned now.
94 Lots of decisions to make. If we are copying less than a cache
95 line we won't be here long. If we are not on a cache line
96 boundary we need to get there. And then we need to figure out
97 how many cache lines ahead to pre-touch.
99 r3 - return value (always)
100 r4 - current source addr
102 r6 - current dest addr
112 /* Establishes GOT addressability so we can load the cache line size
113 from rtld_global_ro. This value was set from the aux vector during
115 SETUP_GOT_ACCESS(r9,got_label)
116 addis r9,r9,_GLOBAL_OFFSET_TABLE_-got_label@ha
117 addi r9,r9,_GLOBAL_OFFSET_TABLE_-got_label@l
120 __GLRO(r9, r9, _dl_cache_line_size,
121 RTLD_GLOBAL_RO_DL_CACHE_LINE_SIZE_OFFSET)
124 bne+ cr5,L(cachelineset)
126 /* Cache line size not set: generic byte copy without much optimization */
127 andi. r0,r5,1 /* If length is odd copy one byte. */
128 beq L(cachelinenotset_align)
129 lbz r7,0(r4) /* Read one byte from source. */
130 addi r5,r5,-1 /* Update length. */
131 addi r4,r4,1 /* Update source pointer address. */
132 stb r7,0(r6) /* Store one byte on dest. */
133 addi r6,r6,1 /* Update dest pointer address. */
134 L(cachelinenotset_align):
135 cmpwi cr7,r5,0 /* If length is 0 return. */
137 ori r2,r2,0 /* Force a new dispatch group. */
138 L(cachelinenotset_loop):
139 addic. r5,r5,-2 /* Update length. */
140 lbz r7,0(r4) /* Load 2 bytes from source. */
142 addi r4,r4,2 /* Update source pointer address. */
143 stb r7,0(r6) /* Store 2 bytes on dest. */
145 addi r6,r6,2 /* Update dest pointer address. */
146 bne L(cachelinenotset_loop)
154 cmpw cr5,r5,r10 /* Less than a cacheline to go? */
156 neg r7,r6 /* How far to next cacheline bdy? */
158 addi r6,r6,-8 /* prepare for stdu */
160 addi r4,r4,-8 /* prepare for ldu */
163 ble+ cr5,L(lessthancacheline)
165 beq- cr0,L(big_lines) /* 128 byte line code */
170 /* More than a cacheline left to go, and using 64 byte cachelines */
172 clrlwi r7,r7,32-6 /* How far to next cacheline bdy? */
174 cmplwi cr6,r7,0 /* Are we on a cacheline bdy already? */
176 /* Reduce total len by what it takes to get to the next cache line */
178 srwi r7,r7,4 /* How many qws to get to the line bdy? */
180 /* How many full cache lines to copy after getting to a line bdy? */
183 cmplwi r10,0 /* If no full cache lines to copy ... */
184 li r11,0 /* number cachelines to copy with prefetch */
185 beq L(nocacheprefetch)
188 /* We are here because we have at least one full cache line to copy,
189 and therefore some pre-touching to do. */
191 cmplwi r10,PREFETCH_AHEAD
192 li r12,64+8 /* prefetch distance */
193 ble L(lessthanmaxprefetch)
195 /* We can only do so much pre-fetching. R11 will have the count of
196 lines left to prefetch after the initial batch of prefetches
199 subi r11,r10,PREFETCH_AHEAD
200 li r10,PREFETCH_AHEAD
202 L(lessthanmaxprefetch):
205 /* At this point r10/ctr hold the number of lines to prefetch in this
206 initial batch, and r11 holds any remainder. */
214 /* Prefetching is done, or was not needed.
216 cr6 - are we on a cacheline boundary already?
217 r7 - number of quadwords to the next cacheline boundary
223 cmplwi cr1,r5,64 /* Less than a cache line to copy? */
225 /* How many bytes are left after we copy whatever full
226 cache lines we can get? */
229 beq cr6,L(cachelinealigned)
232 /* Copy quadwords up to the next cacheline boundary */
239 bdnz L(aligntocacheline)
243 L(cachelinealigned): /* copy while cache lines */
245 blt- cr1,L(lessthancacheline) /* size <64 */
252 li r11,64*ZERO_AHEAD +8 /* DCBZ dist */
255 /* Copy whole cachelines, optimized by prefetching SRC cacheline */
256 L(loop): /* Copy aligned body */
257 dcbt r12,r4 /* PREFETCH SOURCE some cache lines ahead */
284 L(loop2): /* Copy aligned body */
307 L(lessthancacheline): /* Was there less than cache to do ? */
309 srwi r7,r5,4 /* divide size by 16 */
318 bdnz L(copy_remaining)
320 L(do_lt16): /* less than 16 ? */
321 cmplwi cr0,r5,0 /* copy remaining bytes (0-15) */
322 beqlr+ /* no rest to copy */
326 L(shortcopy): /* SIMPLE COPY to handle size =< 15 bytes */
330 lfdx fp9,r7,r6 /* copy 8 byte */
335 lwzx r0,r7,r6 /* copy 4 byte */
340 lhzx r0,r7,r6 /* copy 2 byte */
345 lbzx r0,r7,r6 /* copy 1 byte */
354 /* Similar to above, but for use with 128 byte lines. */
359 clrlwi r7,r7,32-7 /* How far to next cacheline bdy? */
361 cmplwi cr6,r7,0 /* Are we on a cacheline bdy already? */
363 /* Reduce total len by what it takes to get to the next cache line */
365 srwi r7,r7,4 /* How many qw to get to the line bdy? */
367 /* How many full cache lines to copy after getting to a line bdy? */
370 cmplwi r10,0 /* If no full cache lines to copy ... */
371 li r11,0 /* number cachelines to copy with prefetch */
372 beq L(nocacheprefetch_128)
375 /* We are here because we have at least one full cache line to copy,
376 and therefore some pre-touching to do. */
378 cmplwi r10,PREFETCH_AHEAD
379 li r12,128+8 /* prefetch distance */
380 ble L(lessthanmaxprefetch_128)
382 /* We can only do so much pre-fetching. R11 will have the count of
383 lines left to prefetch after the initial batch of prefetches
386 subi r11,r10,PREFETCH_AHEAD
387 li r10,PREFETCH_AHEAD
389 L(lessthanmaxprefetch_128):
392 /* At this point r10/ctr hold the number of lines to prefetch in this
393 initial batch, and r11 holds any remainder. */
398 bdnz L(prefetchSRC_128)
401 /* Prefetching is done, or was not needed.
403 cr6 - are we on a cacheline boundary already?
404 r7 - number of quadwords to the next cacheline boundary
407 L(nocacheprefetch_128):
410 cmplwi cr1,r5,128 /* Less than a cache line to copy? */
412 /* How many bytes are left after we copy whatever full
413 cache lines we can get? */
416 beq cr6,L(cachelinealigned_128)
419 /* Copy quadwords up to the next cacheline boundary */
421 L(aligntocacheline_128):
426 bdnz L(aligntocacheline_128)
429 L(cachelinealigned_128): /* copy while cache lines */
431 blt- cr1,L(lessthancacheline) /* size <128 */
438 li r11,128*ZERO_AHEAD +8 /* DCBZ dist */
441 /* Copy whole cachelines, optimized by prefetching SRC cacheline */
442 L(loop_128): /* Copy aligned body */
443 dcbt r12,r4 /* PREFETCH SOURCE some cache lines ahead */
486 L(loop2_128): /* Copy aligned body */
522 b L(lessthancacheline)
526 libc_hidden_builtin_def (memcpy)