| blob | 2420f931b25ad3b0c6f975122f685a479be1a39d |
1 /* Copyright (C) 2012-2014 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library. If not, see
16 <http://www.gnu.org/licenses/>. */
18 #ifdef ANDROID_CHANGES
19 #include "machine/asm.h"
20 #include "machine/regdef.h"
21 #define USE_MEMMOVE_FOR_OVERLAP
22 #define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
23 #define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
24 #elif _LIBC
25 #include <sysdep.h>
26 #include <regdef.h>
27 #include <sys/asm.h>
28 #define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
29 #define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
30 #elif _COMPILING_NEWLIB
31 #include "machine/asm.h"
32 #include "machine/regdef.h"
33 #define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
34 #define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
35 #else
36 #include <regdef.h>
37 #include <sys/asm.h>
38 #endif
40 #if (_MIPS_ISA == _MIPS_ISA_MIPS4) || (_MIPS_ISA == _MIPS_ISA_MIPS5) || \
41 (_MIPS_ISA == _MIPS_ISA_MIPS32) || (_MIPS_ISA == _MIPS_ISA_MIPS64)
42 #ifndef DISABLE_PREFETCH
43 #define USE_PREFETCH
44 #endif
45 #endif
47 #if defined(_MIPS_SIM) && ((_MIPS_SIM == _ABI64) || (_MIPS_SIM == _ABIN32))
48 #ifndef DISABLE_DOUBLE
49 #define USE_DOUBLE
50 #endif
51 #endif
55 /* Some asm.h files do not have the L macro definition. */
56 #ifndef L
57 #if _MIPS_SIM == _ABIO32
58 # define L(label) $L ## label
59 #else
60 # define L(label) .L ## label
61 #endif
62 #endif
64 /* Some asm.h files do not have the PTR_ADDIU macro definition. */
65 #ifndef PTR_ADDIU
66 #ifdef USE_DOUBLE
67 #define PTR_ADDIU daddiu
68 #else
69 #define PTR_ADDIU addiu
70 #endif
71 #endif
73 /* Some asm.h files do not have the PTR_SRA macro definition. */
74 #ifndef PTR_SRA
75 #ifdef USE_DOUBLE
76 #define PTR_SRA dsra
77 #else
78 #define PTR_SRA sra
79 #endif
80 #endif
83 /*
84 * Using PREFETCH_HINT_LOAD_STREAMED instead of PREFETCH_LOAD on load
85 * prefetches appears to offer a slight preformance advantage.
86 *
87 * Using PREFETCH_HINT_PREPAREFORSTORE instead of PREFETCH_STORE
88 * or PREFETCH_STORE_STREAMED offers a large performance advantage
89 * but PREPAREFORSTORE has some special restrictions to consider.
90 *
91 * Prefetch with the 'prepare for store' hint does not copy a memory
92 * location into the cache, it just allocates a cache line and zeros
93 * it out. This means that if you do not write to the entire cache
94 * line before writing it out to memory some data will get zero'ed out
95 * when the cache line is written back to memory and data will be lost.
96 *
97 * Also if you are using this memcpy to copy overlapping buffers it may
98 * not behave correctly when using the 'prepare for store' hint. If you
99 * use the 'prepare for store' prefetch on a memory area that is in the
100 * memcpy source (as well as the memcpy destination), then you will get
101 * some data zero'ed out before you have a chance to read it and data will
102 * be lost.
103 *
104 * If you are going to use this memcpy routine with the 'prepare for store'
105 * prefetch you may want to set USE_MEMMOVE_FOR_OVERLAP in order to avoid
106 * the problem of running memcpy on overlapping buffers.
107 *
108 * There are ifdef'ed sections of this memcpy to make sure that it does not
109 * do prefetches on cache lines that are not going to be completely written.
110 * This code is only needed and only used when PREFETCH_STORE_HINT is set to
111 * PREFETCH_HINT_PREPAREFORSTORE. This code assumes that cache lines are
112 * 32 bytes and if the cache line is larger it will not work correctly.
113 */
115 #ifdef USE_PREFETCH
116 # define PREFETCH_HINT_LOAD 0
117 # define PREFETCH_HINT_STORE 1
118 # define PREFETCH_HINT_LOAD_STREAMED 4
119 # define PREFETCH_HINT_STORE_STREAMED 5
120 # define PREFETCH_HINT_LOAD_RETAINED 6
121 # define PREFETCH_HINT_STORE_RETAINED 7
122 # define PREFETCH_HINT_WRITEBACK_INVAL 25
123 # define PREFETCH_HINT_PREPAREFORSTORE 30
125 /*
126 * If we have not picked out what hints to use at this point use the
127 * standard load and store prefetch hints.
128 */
129 #ifndef PREFETCH_STORE_HINT
130 # define PREFETCH_STORE_HINT PREFETCH_HINT_STORE
131 #endif
132 #ifndef PREFETCH_LOAD_HINT
133 # define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD
134 #endif
136 /*
137 * We double everything when USE_DOUBLE is true so we do 2 prefetches to
138 * get 64 bytes in that case. The assumption is that each individual
139 * prefetch brings in 32 bytes.
140 */
142 #ifdef USE_DOUBLE
143 # define PREFETCH_CHUNK 64
144 # define PREFETCH_FOR_LOAD(chunk, reg) \
145 pref PREFETCH_LOAD_HINT, (chunk)*64(reg); \
146 pref PREFETCH_LOAD_HINT, ((chunk)*64)+32(reg)
147 # define PREFETCH_FOR_STORE(chunk, reg) \
148 pref PREFETCH_STORE_HINT, (chunk)*64(reg); \
149 pref PREFETCH_STORE_HINT, ((chunk)*64)+32(reg)
150 #else
151 # define PREFETCH_CHUNK 32
152 # define PREFETCH_FOR_LOAD(chunk, reg) \
153 pref PREFETCH_LOAD_HINT, (chunk)*32(reg)
154 # define PREFETCH_FOR_STORE(chunk, reg) \
155 pref PREFETCH_STORE_HINT, (chunk)*32(reg)
156 #endif
157 /* MAX_PREFETCH_SIZE is the maximum size of a prefetch, it must not be less
158 * than PREFETCH_CHUNK, the assumed size of each prefetch. If the real size
159 * of a prefetch is greater than MAX_PREFETCH_SIZE and the PREPAREFORSTORE
160 * hint is used, the code will not work correctly. If PREPAREFORSTORE is not
161 * used then MAX_PREFETCH_SIZE does not matter. */
162 #define MAX_PREFETCH_SIZE 128
163 /* PREFETCH_LIMIT is set based on the fact that we never use an offset greater
164 * than 5 on a STORE prefetch and that a single prefetch can never be larger
165 * than MAX_PREFETCH_SIZE. We add the extra 32 when USE_DOUBLE is set because
166 * we actually do two prefetches in that case, one 32 bytes after the other. */
167 #ifdef USE_DOUBLE
168 # define PREFETCH_LIMIT (5 * PREFETCH_CHUNK) + 32 + MAX_PREFETCH_SIZE
169 #else
170 # define PREFETCH_LIMIT (5 * PREFETCH_CHUNK) + MAX_PREFETCH_SIZE
171 #endif
172 #if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE) \
173 && ((PREFETCH_CHUNK * 4) < MAX_PREFETCH_SIZE)
174 /* We cannot handle this because the initial prefetches may fetch bytes that
175 * are before the buffer being copied. We start copies with an offset
176 * of 4 so avoid this situation when using PREPAREFORSTORE. */
177 #error "PREFETCH_CHUNK is too large and/or MAX_PREFETCH_SIZE is too small."
178 #endif
179 #else /* USE_PREFETCH not defined */
180 # define PREFETCH_FOR_LOAD(offset, reg)
181 # define PREFETCH_FOR_STORE(offset, reg)
182 #endif
184 /* Allow the routine to be named something else if desired. */
185 #ifndef MEMCPY_NAME
186 #define MEMCPY_NAME memcpy
187 #endif
189 /* We use these 32/64 bit registers as temporaries to do the copying. */
190 #define REG0 t0
191 #define REG1 t1
192 #define REG2 t2
193 #define REG3 t3
194 #if defined(_MIPS_SIM) && ((_MIPS_SIM == _ABIO32) || (_MIPS_SIM == _ABIO64))
195 # define REG4 t4
196 # define REG5 t5
197 # define REG6 t6
198 # define REG7 t7
199 #else
200 # define REG4 ta0
201 # define REG5 ta1
202 # define REG6 ta2
203 # define REG7 ta3
204 #endif
206 /* We load/store 64 bits at a time when USE_DOUBLE is true.
207 * The C_ prefix stands for CHUNK and is used to avoid macro name
208 * conflicts with system header files. */
210 #ifdef USE_DOUBLE
211 # define C_ST sd
212 # define C_LD ld
213 #if __MIPSEB
214 # define C_LDHI ldl /* high part is left in big-endian */
215 # define C_STHI sdl /* high part is left in big-endian */
216 # define C_LDLO ldr /* low part is right in big-endian */
217 # define C_STLO sdr /* low part is right in big-endian */
218 #else
219 # define C_LDHI ldr /* high part is right in little-endian */
220 # define C_STHI sdr /* high part is right in little-endian */
221 # define C_LDLO ldl /* low part is left in little-endian */
222 # define C_STLO sdl /* low part is left in little-endian */
223 #endif
224 #else
225 # define C_ST sw
226 # define C_LD lw
227 #if __MIPSEB
228 # define C_LDHI lwl /* high part is left in big-endian */
229 # define C_STHI swl /* high part is left in big-endian */
230 # define C_LDLO lwr /* low part is right in big-endian */
231 # define C_STLO swr /* low part is right in big-endian */
232 #else
233 # define C_LDHI lwr /* high part is right in little-endian */
234 # define C_STHI swr /* high part is right in little-endian */
235 # define C_LDLO lwl /* low part is left in little-endian */
236 # define C_STLO swl /* low part is left in little-endian */
237 #endif
238 #endif
240 /* Bookkeeping values for 32 vs. 64 bit mode. */
241 #ifdef USE_DOUBLE
242 # define NSIZE 8
243 # define NSIZEMASK 0x3f
244 # define NSIZEDMASK 0x7f
245 #else
246 # define NSIZE 4
247 # define NSIZEMASK 0x1f
248 # define NSIZEDMASK 0x3f
249 #endif
250 #define UNIT(unit) ((unit)*NSIZE)
251 #define UNITM1(unit) (((unit)*NSIZE)-1)
253 #ifdef ANDROID_CHANGES
254 LEAF(MEMCPY_NAME, 0)
255 #else
256 LEAF(MEMCPY_NAME)
257 #endif
258 .set nomips16
259 .set noreorder
260 /*
261 * Below we handle the case where memcpy is called with overlapping src and dst.
262 * Although memcpy is not required to handle this case, some parts of Android
263 * like Skia rely on such usage. We call memmove to handle such cases.
264 */
265 #ifdef USE_MEMMOVE_FOR_OVERLAP
266 PTR_SUBU t0,a0,a1
267 PTR_SRA t2,t0,31
268 xor t1,t0,t2
269 PTR_SUBU t0,t1,t2
270 sltu t2,t0,a2
271 beq t2,zero,L(memcpy)
272 la t9,memmove
273 jr t9
274 nop
275 L(memcpy):
276 #endif
277 /*
278 * If the size is less than 2*NSIZE (8 or 16), go to L(lastb). Regardless of
279 * size, copy dst pointer to v0 for the return value.
280 */
281 slti t2,a2,(2 * NSIZE)
282 bne t2,zero,L(lastb)
283 #if defined(RETURN_FIRST_PREFETCH) || defined(RETURN_LAST_PREFETCH)
284 move v0,zero
285 #else
286 move v0,a0
287 #endif
288 /*
289 * If src and dst have different alignments, go to L(unaligned), if they
290 * have the same alignment (but are not actually aligned) do a partial
291 * load/store to make them aligned. If they are both already aligned
292 * we can start copying at L(aligned).
293 */
294 xor t8,a1,a0
295 andi t8,t8,(NSIZE-1) /* t8 is a0/a1 word-displacement */
296 bne t8,zero,L(unaligned)
297 PTR_SUBU a3, zero, a0
299 andi a3,a3,(NSIZE-1) /* copy a3 bytes to align a0/a1 */
300 beq a3,zero,L(aligned) /* if a3=0, it is already aligned */
301 PTR_SUBU a2,a2,a3 /* a2 is the remining bytes count */
303 C_LDHI t8,0(a1)
304 PTR_ADDU a1,a1,a3
305 C_STHI t8,0(a0)
306 PTR_ADDU a0,a0,a3
308 /*
309 * Now dst/src are both aligned to (word or double word) aligned addresses
310 * Set a2 to count how many bytes we have to copy after all the 64/128 byte
311 * chunks are copied and a3 to the dst pointer after all the 64/128 byte
312 * chunks have been copied. We will loop, incrementing a0 and a1 until a0
313 * equals a3.
314 */
316 L(aligned):
317 andi t8,a2,NSIZEDMASK /* any whole 64-byte/128-byte chunks? */
318 beq a2,t8,L(chkw) /* if a2==t8, no 64-byte/128-byte chunks */
319 PTR_SUBU a3,a2,t8 /* subtract from a2 the reminder */
320 PTR_ADDU a3,a0,a3 /* Now a3 is the final dst after loop */
322 /* When in the loop we may prefetch with the 'prepare to store' hint,
323 * in this case the a0+x should not be past the "t0-32" address. This
324 * means: for x=128 the last "safe" a0 address is "t0-160". Alternatively,
325 * for x=64 the last "safe" a0 address is "t0-96" In the current version we
326 * will use "prefetch hint,128(a0)", so "t0-160" is the limit.
327 */
328 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
329 PTR_ADDU t0,a0,a2 /* t0 is the "past the end" address */
330 PTR_SUBU t9,t0,PREFETCH_LIMIT /* t9 is the "last safe pref" address */
331 #endif
332 PREFETCH_FOR_LOAD (0, a1)
333 PREFETCH_FOR_LOAD (1, a1)
334 PREFETCH_FOR_LOAD (2, a1)
335 PREFETCH_FOR_LOAD (3, a1)
336 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
337 PREFETCH_FOR_STORE (1, a0)
338 PREFETCH_FOR_STORE (2, a0)
339 PREFETCH_FOR_STORE (3, a0)
340 #endif
341 #if defined(RETURN_FIRST_PREFETCH) && defined(USE_PREFETCH)
342 #if PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE
343 sltu v1,t9,a0
344 bgtz v1,L(skip_set)
345 nop
346 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*4)
347 L(skip_set):
348 #else
349 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*1)
350 #endif
351 #endif
352 #if defined(RETURN_LAST_PREFETCH) && defined(USE_PREFETCH) \
353 && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
354 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*3)
355 #ifdef USE_DOUBLE
356 PTR_ADDIU v0,v0,32
357 #endif
358 #endif
359 L(loop16w):
360 C_LD t0,UNIT(0)(a1)
361 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
362 sltu v1,t9,a0 /* If a0 > t9 don't use next prefetch */
363 bgtz v1,L(skip_pref)
364 #endif
365 C_LD t1,UNIT(1)(a1)
366 PREFETCH_FOR_STORE (4, a0)
367 PREFETCH_FOR_STORE (5, a0)
368 #if defined(RETURN_LAST_PREFETCH) && defined(USE_PREFETCH)
369 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*5)
370 #ifdef USE_DOUBLE
371 PTR_ADDIU v0,v0,32
372 #endif
373 #endif
374 L(skip_pref):
375 C_LD REG2,UNIT(2)(a1)
376 C_LD REG3,UNIT(3)(a1)
377 C_LD REG4,UNIT(4)(a1)
378 C_LD REG5,UNIT(5)(a1)
379 C_LD REG6,UNIT(6)(a1)
380 C_LD REG7,UNIT(7)(a1)
381 PREFETCH_FOR_LOAD (4, a1)
383 C_ST t0,UNIT(0)(a0)
384 C_ST t1,UNIT(1)(a0)
385 C_ST REG2,UNIT(2)(a0)
386 C_ST REG3,UNIT(3)(a0)
387 C_ST REG4,UNIT(4)(a0)
388 C_ST REG5,UNIT(5)(a0)
389 C_ST REG6,UNIT(6)(a0)
390 C_ST REG7,UNIT(7)(a0)
392 C_LD t0,UNIT(8)(a1)
393 C_LD t1,UNIT(9)(a1)
394 C_LD REG2,UNIT(10)(a1)
395 C_LD REG3,UNIT(11)(a1)
396 C_LD REG4,UNIT(12)(a1)
397 C_LD REG5,UNIT(13)(a1)
398 C_LD REG6,UNIT(14)(a1)
399 C_LD REG7,UNIT(15)(a1)
400 PREFETCH_FOR_LOAD (5, a1)
401 C_ST t0,UNIT(8)(a0)
402 C_ST t1,UNIT(9)(a0)
403 C_ST REG2,UNIT(10)(a0)
404 C_ST REG3,UNIT(11)(a0)
405 C_ST REG4,UNIT(12)(a0)
406 C_ST REG5,UNIT(13)(a0)
407 C_ST REG6,UNIT(14)(a0)
408 C_ST REG7,UNIT(15)(a0)
409 PTR_ADDIU a0,a0,UNIT(16) /* adding 64/128 to dest */
410 bne a0,a3,L(loop16w)
411 PTR_ADDIU a1,a1,UNIT(16) /* adding 64/128 to src */
412 move a2,t8
414 /* Here we have src and dest word-aligned but less than 64-bytes or
415 * 128 bytes to go. Check for a 32(64) byte chunk and copy if if there
416 * is one. Otherwise jump down to L(chk1w) to handle the tail end of
417 * the copy.
418 */
420 L(chkw):
421 PREFETCH_FOR_LOAD (0, a1)
422 andi t8,a2,NSIZEMASK /* Is there a 32-byte/64-byte chunk. */
423 /* The t8 is the reminder count past 32-bytes */
424 beq a2,t8,L(chk1w) /* When a2=t8, no 32-byte chunk */
425 nop
426 C_LD t0,UNIT(0)(a1)
427 C_LD t1,UNIT(1)(a1)
428 C_LD REG2,UNIT(2)(a1)
429 C_LD REG3,UNIT(3)(a1)
430 C_LD REG4,UNIT(4)(a1)
431 C_LD REG5,UNIT(5)(a1)
432 C_LD REG6,UNIT(6)(a1)
433 C_LD REG7,UNIT(7)(a1)
434 PTR_ADDIU a1,a1,UNIT(8)
435 C_ST t0,UNIT(0)(a0)
436 C_ST t1,UNIT(1)(a0)
437 C_ST REG2,UNIT(2)(a0)
438 C_ST REG3,UNIT(3)(a0)
439 C_ST REG4,UNIT(4)(a0)
440 C_ST REG5,UNIT(5)(a0)
441 C_ST REG6,UNIT(6)(a0)
442 C_ST REG7,UNIT(7)(a0)
443 PTR_ADDIU a0,a0,UNIT(8)
445 /*
446 * Here we have less than 32(64) bytes to copy. Set up for a loop to
447 * copy one word (or double word) at a time. Set a2 to count how many
448 * bytes we have to copy after all the word (or double word) chunks are
449 * copied and a3 to the dst pointer after all the (d)word chunks have
450 * been copied. We will loop, incrementing a0 and a1 until a0 equals a3.
451 */
452 L(chk1w):
453 andi a2,t8,(NSIZE-1) /* a2 is the reminder past one (d)word chunks */
454 beq a2,t8,L(lastb)
455 PTR_SUBU a3,t8,a2 /* a3 is count of bytes in one (d)word chunks */
456 PTR_ADDU a3,a0,a3 /* a3 is the dst address after loop */
458 /* copying in words (4-byte or 8-byte chunks) */
459 L(wordCopy_loop):
460 C_LD REG3,UNIT(0)(a1)
461 PTR_ADDIU a0,a0,UNIT(1)
462 PTR_ADDIU a1,a1,UNIT(1)
463 bne a0,a3,L(wordCopy_loop)
464 C_ST REG3,UNIT(-1)(a0)
466 /* Copy the last 8 (or 16) bytes */
467 L(lastb):
468 blez a2,L(leave)
469 PTR_ADDU a3,a0,a2 /* a3 is the last dst address */
470 L(lastbloop):
471 lb v1,0(a1)
472 PTR_ADDIU a0,a0,1
473 PTR_ADDIU a1,a1,1
474 bne a0,a3,L(lastbloop)
475 sb v1,-1(a0)
476 L(leave):
477 j ra
478 nop
479 /*
480 * UNALIGNED case, got here with a3 = "negu a0"
481 * This code is nearly identical to the aligned code above
482 * but only the destination (not the source) gets aligned
483 * so we need to do partial loads of the source followed
484 * by normal stores to the destination (once we have aligned
485 * the destination).
486 */
488 L(unaligned):
489 andi a3,a3,(NSIZE-1) /* copy a3 bytes to align a0/a1 */
490 beqz a3,L(ua_chk16w) /* if a3=0, it is already aligned */
491 PTR_SUBU a2,a2,a3 /* a2 is the remining bytes count */
493 C_LDHI v1,UNIT(0)(a1)
494 C_LDLO v1,UNITM1(1)(a1)
495 PTR_ADDU a1,a1,a3
496 C_STHI v1,UNIT(0)(a0)
497 PTR_ADDU a0,a0,a3
499 /*
500 * Now the destination (but not the source) is aligned
501 * Set a2 to count how many bytes we have to copy after all the 64/128 byte
502 * chunks are copied and a3 to the dst pointer after all the 64/128 byte
503 * chunks have been copied. We will loop, incrementing a0 and a1 until a0
504 * equals a3.
505 */
507 L(ua_chk16w):
508 andi t8,a2,NSIZEDMASK /* any whole 64-byte/128-byte chunks? */
509 beq a2,t8,L(ua_chkw) /* if a2==t8, no 64-byte/128-byte chunks */
510 PTR_SUBU a3,a2,t8 /* subtract from a2 the reminder */
511 PTR_ADDU a3,a0,a3 /* Now a3 is the final dst after loop */
513 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
514 PTR_ADDU t0,a0,a2 /* t0 is the "past the end" address */
515 PTR_SUBU t9,t0,PREFETCH_LIMIT /* t9 is the "last safe pref" address */
516 #endif
517 PREFETCH_FOR_LOAD (0, a1)
518 PREFETCH_FOR_LOAD (1, a1)
519 PREFETCH_FOR_LOAD (2, a1)
520 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
521 PREFETCH_FOR_STORE (1, a0)
522 PREFETCH_FOR_STORE (2, a0)
523 PREFETCH_FOR_STORE (3, a0)
524 #endif
525 #if defined(RETURN_FIRST_PREFETCH) && defined(USE_PREFETCH)
526 #if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
527 sltu v1,t9,a0
528 bgtz v1,L(ua_skip_set)
529 nop
530 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*4)
531 L(ua_skip_set):
532 #else
533 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*1)
534 #endif
535 #endif
536 L(ua_loop16w):
537 PREFETCH_FOR_LOAD (3, a1)
538 C_LDHI t0,UNIT(0)(a1)
539 C_LDHI t1,UNIT(1)(a1)
540 C_LDHI REG2,UNIT(2)(a1)
541 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
542 sltu v1,t9,a0
543 bgtz v1,L(ua_skip_pref)
544 #endif
545 C_LDHI REG3,UNIT(3)(a1)
546 PREFETCH_FOR_STORE (4, a0)
547 PREFETCH_FOR_STORE (5, a0)
548 L(ua_skip_pref):
549 C_LDHI REG4,UNIT(4)(a1)
550 C_LDHI REG5,UNIT(5)(a1)
551 C_LDHI REG6,UNIT(6)(a1)
552 C_LDHI REG7,UNIT(7)(a1)
553 C_LDLO t0,UNITM1(1)(a1)
554 C_LDLO t1,UNITM1(2)(a1)
555 C_LDLO REG2,UNITM1(3)(a1)
556 C_LDLO REG3,UNITM1(4)(a1)
557 C_LDLO REG4,UNITM1(5)(a1)
558 C_LDLO REG5,UNITM1(6)(a1)
559 C_LDLO REG6,UNITM1(7)(a1)
560 C_LDLO REG7,UNITM1(8)(a1)
561 PREFETCH_FOR_LOAD (4, a1)
562 C_ST t0,UNIT(0)(a0)
563 C_ST t1,UNIT(1)(a0)
564 C_ST REG2,UNIT(2)(a0)
565 C_ST REG3,UNIT(3)(a0)
566 C_ST REG4,UNIT(4)(a0)
567 C_ST REG5,UNIT(5)(a0)
568 C_ST REG6,UNIT(6)(a0)
569 C_ST REG7,UNIT(7)(a0)
570 C_LDHI t0,UNIT(8)(a1)
571 C_LDHI t1,UNIT(9)(a1)
572 C_LDHI REG2,UNIT(10)(a1)
573 C_LDHI REG3,UNIT(11)(a1)
574 C_LDHI REG4,UNIT(12)(a1)
575 C_LDHI REG5,UNIT(13)(a1)
576 C_LDHI REG6,UNIT(14)(a1)
577 C_LDHI REG7,UNIT(15)(a1)
578 C_LDLO t0,UNITM1(9)(a1)
579 C_LDLO t1,UNITM1(10)(a1)
580 C_LDLO REG2,UNITM1(11)(a1)
581 C_LDLO REG3,UNITM1(12)(a1)
582 C_LDLO REG4,UNITM1(13)(a1)
583 C_LDLO REG5,UNITM1(14)(a1)
584 C_LDLO REG6,UNITM1(15)(a1)
585 C_LDLO REG7,UNITM1(16)(a1)
586 PREFETCH_FOR_LOAD (5, a1)
587 C_ST t0,UNIT(8)(a0)
588 C_ST t1,UNIT(9)(a0)
589 C_ST REG2,UNIT(10)(a0)
590 C_ST REG3,UNIT(11)(a0)
591 C_ST REG4,UNIT(12)(a0)
592 C_ST REG5,UNIT(13)(a0)
593 C_ST REG6,UNIT(14)(a0)
594 C_ST REG7,UNIT(15)(a0)
595 PTR_ADDIU a0,a0,UNIT(16) /* adding 64/128 to dest */
596 bne a0,a3,L(ua_loop16w)
597 PTR_ADDIU a1,a1,UNIT(16) /* adding 64/128 to src */
598 move a2,t8
600 /* Here we have src and dest word-aligned but less than 64-bytes or
601 * 128 bytes to go. Check for a 32(64) byte chunk and copy if if there
602 * is one. Otherwise jump down to L(ua_chk1w) to handle the tail end of
603 * the copy. */
605 L(ua_chkw):
606 PREFETCH_FOR_LOAD (0, a1)
607 andi t8,a2,NSIZEMASK /* Is there a 32-byte/64-byte chunk. */
608 /* t8 is the reminder count past 32-bytes */
609 beq a2,t8,L(ua_chk1w) /* When a2=t8, no 32-byte chunk */
610 nop
611 C_LDHI t0,UNIT(0)(a1)
612 C_LDHI t1,UNIT(1)(a1)
613 C_LDHI REG2,UNIT(2)(a1)
614 C_LDHI REG3,UNIT(3)(a1)
615 C_LDHI REG4,UNIT(4)(a1)
616 C_LDHI REG5,UNIT(5)(a1)
617 C_LDHI REG6,UNIT(6)(a1)
618 C_LDHI REG7,UNIT(7)(a1)
619 C_LDLO t0,UNITM1(1)(a1)
620 C_LDLO t1,UNITM1(2)(a1)
621 C_LDLO REG2,UNITM1(3)(a1)
622 C_LDLO REG3,UNITM1(4)(a1)
623 C_LDLO REG4,UNITM1(5)(a1)
624 C_LDLO REG5,UNITM1(6)(a1)
625 C_LDLO REG6,UNITM1(7)(a1)
626 C_LDLO REG7,UNITM1(8)(a1)
627 PTR_ADDIU a1,a1,UNIT(8)
628 C_ST t0,UNIT(0)(a0)
629 C_ST t1,UNIT(1)(a0)
630 C_ST REG2,UNIT(2)(a0)
631 C_ST REG3,UNIT(3)(a0)
632 C_ST REG4,UNIT(4)(a0)
633 C_ST REG5,UNIT(5)(a0)
634 C_ST REG6,UNIT(6)(a0)
635 C_ST REG7,UNIT(7)(a0)
636 PTR_ADDIU a0,a0,UNIT(8)
637 /*
638 * Here we have less than 32(64) bytes to copy. Set up for a loop to
639 * copy one word (or double word) at a time.
640 */
641 L(ua_chk1w):
642 andi a2,t8,(NSIZE-1) /* a2 is the reminder past one (d)word chunks */
643 beq a2,t8,L(ua_smallCopy)
644 PTR_SUBU a3,t8,a2 /* a3 is count of bytes in one (d)word chunks */
645 PTR_ADDU a3,a0,a3 /* a3 is the dst address after loop */
647 /* copying in words (4-byte or 8-byte chunks) */
648 L(ua_wordCopy_loop):
649 C_LDHI v1,UNIT(0)(a1)
650 C_LDLO v1,UNITM1(1)(a1)
651 PTR_ADDIU a0,a0,UNIT(1)
652 PTR_ADDIU a1,a1,UNIT(1)
653 bne a0,a3,L(ua_wordCopy_loop)
654 C_ST v1,UNIT(-1)(a0)
656 /* Copy the last 8 (or 16) bytes */
657 L(ua_smallCopy):
658 beqz a2,L(leave)
659 PTR_ADDU a3,a0,a2 /* a3 is the last dst address */
660 L(ua_smallCopy_loop):
661 lb v1,0(a1)
662 PTR_ADDIU a0,a0,1
663 PTR_ADDIU a1,a1,1
664 bne a0,a3,L(ua_smallCopy_loop)
665 sb v1,-1(a0)
667 j ra
668 nop
670 .set at
671 .set reorder
672 END(MEMCPY_NAME)
673 #ifndef ANDROID_CHANGES
674 #ifdef _LIBC
675 libc_hidden_builtin_def (MEMCPY_NAME)
676 #endif
677 #endif