| blob | fcd7c0368cf8c3cc16d909e5a2f85b6f8a14987f |
1 /* Copyright (C) 2012-2015 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library. If not, see
16 <http://www.gnu.org/licenses/>. */
18 #ifdef ANDROID_CHANGES
19 # include "machine/asm.h"
20 # include "machine/regdef.h"
21 # define USE_MEMMOVE_FOR_OVERLAP
22 # define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
23 # define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
24 #elif _LIBC
25 # include <sysdep.h>
26 # include <regdef.h>
27 # include <sys/asm.h>
28 # define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
29 # define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
30 #elif _COMPILING_NEWLIB
31 # include "machine/asm.h"
32 # include "machine/regdef.h"
33 # define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
34 # define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
35 #else
36 # include <regdef.h>
37 # include <sys/asm.h>
38 #endif
40 #if (_MIPS_ISA == _MIPS_ISA_MIPS4) || (_MIPS_ISA == _MIPS_ISA_MIPS5) || \
41 (_MIPS_ISA == _MIPS_ISA_MIPS32) || (_MIPS_ISA == _MIPS_ISA_MIPS64)
42 # ifndef DISABLE_PREFETCH
43 # define USE_PREFETCH
44 # endif
45 #endif
47 #if defined(_MIPS_SIM) && ((_MIPS_SIM == _ABI64) || (_MIPS_SIM == _ABIN32))
48 # ifndef DISABLE_DOUBLE
49 # define USE_DOUBLE
50 # endif
51 #endif
54 #if __mips_isa_rev > 5
55 # if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
56 # undef PREFETCH_STORE_HINT
57 # define PREFETCH_STORE_HINT PREFETCH_HINT_STORE_STREAMED
58 # endif
59 # define R6_CODE
60 #endif
62 /* Some asm.h files do not have the L macro definition. */
63 #ifndef L
64 # if _MIPS_SIM == _ABIO32
65 # define L(label) $L ## label
66 # else
67 # define L(label) .L ## label
68 # endif
69 #endif
71 /* Some asm.h files do not have the PTR_ADDIU macro definition. */
72 #ifndef PTR_ADDIU
73 # ifdef USE_DOUBLE
74 # define PTR_ADDIU daddiu
75 # else
76 # define PTR_ADDIU addiu
77 # endif
78 #endif
80 /* Some asm.h files do not have the PTR_SRA macro definition. */
81 #ifndef PTR_SRA
82 # ifdef USE_DOUBLE
83 # define PTR_SRA dsra
84 # else
85 # define PTR_SRA sra
86 # endif
87 #endif
89 /* New R6 instructions that may not be in asm.h. */
90 #ifndef PTR_LSA
91 # if _MIPS_SIM == _ABI64
92 # define PTR_LSA dlsa
93 # else
94 # define PTR_LSA lsa
95 # endif
96 #endif
98 /*
99 * Using PREFETCH_HINT_LOAD_STREAMED instead of PREFETCH_LOAD on load
100 * prefetches appears to offer a slight preformance advantage.
101 *
102 * Using PREFETCH_HINT_PREPAREFORSTORE instead of PREFETCH_STORE
103 * or PREFETCH_STORE_STREAMED offers a large performance advantage
104 * but PREPAREFORSTORE has some special restrictions to consider.
105 *
106 * Prefetch with the 'prepare for store' hint does not copy a memory
107 * location into the cache, it just allocates a cache line and zeros
108 * it out. This means that if you do not write to the entire cache
109 * line before writing it out to memory some data will get zero'ed out
110 * when the cache line is written back to memory and data will be lost.
111 *
112 * Also if you are using this memcpy to copy overlapping buffers it may
113 * not behave correctly when using the 'prepare for store' hint. If you
114 * use the 'prepare for store' prefetch on a memory area that is in the
115 * memcpy source (as well as the memcpy destination), then you will get
116 * some data zero'ed out before you have a chance to read it and data will
117 * be lost.
118 *
119 * If you are going to use this memcpy routine with the 'prepare for store'
120 * prefetch you may want to set USE_MEMMOVE_FOR_OVERLAP in order to avoid
121 * the problem of running memcpy on overlapping buffers.
122 *
123 * There are ifdef'ed sections of this memcpy to make sure that it does not
124 * do prefetches on cache lines that are not going to be completely written.
125 * This code is only needed and only used when PREFETCH_STORE_HINT is set to
126 * PREFETCH_HINT_PREPAREFORSTORE. This code assumes that cache lines are
127 * 32 bytes and if the cache line is larger it will not work correctly.
128 */
130 #ifdef USE_PREFETCH
131 # define PREFETCH_HINT_LOAD 0
132 # define PREFETCH_HINT_STORE 1
133 # define PREFETCH_HINT_LOAD_STREAMED 4
134 # define PREFETCH_HINT_STORE_STREAMED 5
135 # define PREFETCH_HINT_LOAD_RETAINED 6
136 # define PREFETCH_HINT_STORE_RETAINED 7
137 # define PREFETCH_HINT_WRITEBACK_INVAL 25
138 # define PREFETCH_HINT_PREPAREFORSTORE 30
140 /*
141 * If we have not picked out what hints to use at this point use the
142 * standard load and store prefetch hints.
143 */
144 # ifndef PREFETCH_STORE_HINT
145 # define PREFETCH_STORE_HINT PREFETCH_HINT_STORE
146 # endif
147 # ifndef PREFETCH_LOAD_HINT
148 # define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD
149 # endif
151 /*
152 * We double everything when USE_DOUBLE is true so we do 2 prefetches to
153 * get 64 bytes in that case. The assumption is that each individual
154 * prefetch brings in 32 bytes.
155 */
157 # ifdef USE_DOUBLE
158 # define PREFETCH_CHUNK 64
159 # define PREFETCH_FOR_LOAD(chunk, reg) \
160 pref PREFETCH_LOAD_HINT, (chunk)*64(reg); \
161 pref PREFETCH_LOAD_HINT, ((chunk)*64)+32(reg)
162 # define PREFETCH_FOR_STORE(chunk, reg) \
163 pref PREFETCH_STORE_HINT, (chunk)*64(reg); \
164 pref PREFETCH_STORE_HINT, ((chunk)*64)+32(reg)
165 # else
166 # define PREFETCH_CHUNK 32
167 # define PREFETCH_FOR_LOAD(chunk, reg) \
168 pref PREFETCH_LOAD_HINT, (chunk)*32(reg)
169 # define PREFETCH_FOR_STORE(chunk, reg) \
170 pref PREFETCH_STORE_HINT, (chunk)*32(reg)
171 # endif
172 /* MAX_PREFETCH_SIZE is the maximum size of a prefetch, it must not be less
173 * than PREFETCH_CHUNK, the assumed size of each prefetch. If the real size
174 * of a prefetch is greater than MAX_PREFETCH_SIZE and the PREPAREFORSTORE
175 * hint is used, the code will not work correctly. If PREPAREFORSTORE is not
176 * used then MAX_PREFETCH_SIZE does not matter. */
177 # define MAX_PREFETCH_SIZE 128
178 /* PREFETCH_LIMIT is set based on the fact that we never use an offset greater
179 * than 5 on a STORE prefetch and that a single prefetch can never be larger
180 * than MAX_PREFETCH_SIZE. We add the extra 32 when USE_DOUBLE is set because
181 * we actually do two prefetches in that case, one 32 bytes after the other. */
182 # ifdef USE_DOUBLE
183 # define PREFETCH_LIMIT (5 * PREFETCH_CHUNK) + 32 + MAX_PREFETCH_SIZE
184 # else
185 # define PREFETCH_LIMIT (5 * PREFETCH_CHUNK) + MAX_PREFETCH_SIZE
186 # endif
187 # if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE) \
188 && ((PREFETCH_CHUNK * 4) < MAX_PREFETCH_SIZE)
189 /* We cannot handle this because the initial prefetches may fetch bytes that
190 * are before the buffer being copied. We start copies with an offset
191 * of 4 so avoid this situation when using PREPAREFORSTORE. */
192 #error "PREFETCH_CHUNK is too large and/or MAX_PREFETCH_SIZE is too small."
193 # endif
194 #else /* USE_PREFETCH not defined */
195 # define PREFETCH_FOR_LOAD(offset, reg)
196 # define PREFETCH_FOR_STORE(offset, reg)
197 #endif
199 /* Allow the routine to be named something else if desired. */
200 #ifndef MEMCPY_NAME
201 # define MEMCPY_NAME memcpy
202 #endif
204 /* We use these 32/64 bit registers as temporaries to do the copying. */
205 #define REG0 t0
206 #define REG1 t1
207 #define REG2 t2
208 #define REG3 t3
209 #if defined(_MIPS_SIM) && ((_MIPS_SIM == _ABIO32) || (_MIPS_SIM == _ABIO64))
210 # define REG4 t4
211 # define REG5 t5
212 # define REG6 t6
213 # define REG7 t7
214 #else
215 # define REG4 ta0
216 # define REG5 ta1
217 # define REG6 ta2
218 # define REG7 ta3
219 #endif
221 /* We load/store 64 bits at a time when USE_DOUBLE is true.
222 * The C_ prefix stands for CHUNK and is used to avoid macro name
223 * conflicts with system header files. */
225 #ifdef USE_DOUBLE
226 # define C_ST sd
227 # define C_LD ld
228 # if __MIPSEB
229 # define C_LDHI ldl /* high part is left in big-endian */
230 # define C_STHI sdl /* high part is left in big-endian */
231 # define C_LDLO ldr /* low part is right in big-endian */
232 # define C_STLO sdr /* low part is right in big-endian */
233 # else
234 # define C_LDHI ldr /* high part is right in little-endian */
235 # define C_STHI sdr /* high part is right in little-endian */
236 # define C_LDLO ldl /* low part is left in little-endian */
237 # define C_STLO sdl /* low part is left in little-endian */
238 # endif
239 # define C_ALIGN dalign /* r6 align instruction */
240 #else
241 # define C_ST sw
242 # define C_LD lw
243 # if __MIPSEB
244 # define C_LDHI lwl /* high part is left in big-endian */
245 # define C_STHI swl /* high part is left in big-endian */
246 # define C_LDLO lwr /* low part is right in big-endian */
247 # define C_STLO swr /* low part is right in big-endian */
248 # else
249 # define C_LDHI lwr /* high part is right in little-endian */
250 # define C_STHI swr /* high part is right in little-endian */
251 # define C_LDLO lwl /* low part is left in little-endian */
252 # define C_STLO swl /* low part is left in little-endian */
253 # endif
254 # define C_ALIGN align /* r6 align instruction */
255 #endif
257 /* Bookkeeping values for 32 vs. 64 bit mode. */
258 #ifdef USE_DOUBLE
259 # define NSIZE 8
260 # define NSIZEMASK 0x3f
261 # define NSIZEDMASK 0x7f
262 #else
263 # define NSIZE 4
264 # define NSIZEMASK 0x1f
265 # define NSIZEDMASK 0x3f
266 #endif
267 #define UNIT(unit) ((unit)*NSIZE)
268 #define UNITM1(unit) (((unit)*NSIZE)-1)
270 #ifdef ANDROID_CHANGES
271 LEAF(MEMCPY_NAME, 0)
272 #else
273 LEAF(MEMCPY_NAME)
274 #endif
275 .set nomips16
276 .set noreorder
277 /*
278 * Below we handle the case where memcpy is called with overlapping src and dst.
279 * Although memcpy is not required to handle this case, some parts of Android
280 * like Skia rely on such usage. We call memmove to handle such cases.
281 */
282 #ifdef USE_MEMMOVE_FOR_OVERLAP
283 PTR_SUBU t0,a0,a1
284 PTR_SRA t2,t0,31
285 xor t1,t0,t2
286 PTR_SUBU t0,t1,t2
287 sltu t2,t0,a2
288 beq t2,zero,L(memcpy)
289 la t9,memmove
290 jr t9
291 nop
292 L(memcpy):
293 #endif
294 /*
295 * If the size is less than 2*NSIZE (8 or 16), go to L(lastb). Regardless of
296 * size, copy dst pointer to v0 for the return value.
297 */
298 slti t2,a2,(2 * NSIZE)
299 bne t2,zero,L(lastb)
300 #if defined(RETURN_FIRST_PREFETCH) || defined(RETURN_LAST_PREFETCH)
301 move v0,zero
302 #else
303 move v0,a0
304 #endif
306 #ifndef R6_CODE
308 /*
309 * If src and dst have different alignments, go to L(unaligned), if they
310 * have the same alignment (but are not actually aligned) do a partial
311 * load/store to make them aligned. If they are both already aligned
312 * we can start copying at L(aligned).
313 */
314 xor t8,a1,a0
315 andi t8,t8,(NSIZE-1) /* t8 is a0/a1 word-displacement */
316 bne t8,zero,L(unaligned)
317 PTR_SUBU a3, zero, a0
319 andi a3,a3,(NSIZE-1) /* copy a3 bytes to align a0/a1 */
320 beq a3,zero,L(aligned) /* if a3=0, it is already aligned */
321 PTR_SUBU a2,a2,a3 /* a2 is the remining bytes count */
323 C_LDHI t8,0(a1)
324 PTR_ADDU a1,a1,a3
325 C_STHI t8,0(a0)
326 PTR_ADDU a0,a0,a3
328 #else /* R6_CODE */
330 /*
331 * Align the destination and hope that the source gets aligned too. If it
332 * doesn't we jump to L(r6_unaligned*) to do unaligned copies using the r6
333 * align instruction.
334 */
335 andi t8,a0,7
336 lapc t9,L(atable)
337 PTR_LSA t9,t8,t9,2
338 jrc t9
339 L(atable):
340 bc L(lb0)
341 bc L(lb7)
342 bc L(lb6)
343 bc L(lb5)
344 bc L(lb4)
345 bc L(lb3)
346 bc L(lb2)
347 bc L(lb1)
348 L(lb7):
349 lb a3, 6(a1)
350 sb a3, 6(a0)
351 L(lb6):
352 lb a3, 5(a1)
353 sb a3, 5(a0)
354 L(lb5):
355 lb a3, 4(a1)
356 sb a3, 4(a0)
357 L(lb4):
358 lb a3, 3(a1)
359 sb a3, 3(a0)
360 L(lb3):
361 lb a3, 2(a1)
362 sb a3, 2(a0)
363 L(lb2):
364 lb a3, 1(a1)
365 sb a3, 1(a0)
366 L(lb1):
367 lb a3, 0(a1)
368 sb a3, 0(a0)
370 li t9,8
371 subu t8,t9,t8
372 PTR_SUBU a2,a2,t8
373 PTR_ADDU a0,a0,t8
374 PTR_ADDU a1,a1,t8
375 L(lb0):
377 andi t8,a1,(NSIZE-1)
378 lapc t9,L(jtable)
379 PTR_LSA t9,t8,t9,2
380 jrc t9
381 L(jtable):
382 bc L(aligned)
383 bc L(r6_unaligned1)
384 bc L(r6_unaligned2)
385 bc L(r6_unaligned3)
386 # ifdef USE_DOUBLE
387 bc L(r6_unaligned4)
388 bc L(r6_unaligned5)
389 bc L(r6_unaligned6)
390 bc L(r6_unaligned7)
391 # endif
392 #endif /* R6_CODE */
394 L(aligned):
396 /*
397 * Now dst/src are both aligned to (word or double word) aligned addresses
398 * Set a2 to count how many bytes we have to copy after all the 64/128 byte
399 * chunks are copied and a3 to the dst pointer after all the 64/128 byte
400 * chunks have been copied. We will loop, incrementing a0 and a1 until a0
401 * equals a3.
402 */
404 andi t8,a2,NSIZEDMASK /* any whole 64-byte/128-byte chunks? */
405 beq a2,t8,L(chkw) /* if a2==t8, no 64-byte/128-byte chunks */
406 PTR_SUBU a3,a2,t8 /* subtract from a2 the reminder */
407 PTR_ADDU a3,a0,a3 /* Now a3 is the final dst after loop */
409 /* When in the loop we may prefetch with the 'prepare to store' hint,
410 * in this case the a0+x should not be past the "t0-32" address. This
411 * means: for x=128 the last "safe" a0 address is "t0-160". Alternatively,
412 * for x=64 the last "safe" a0 address is "t0-96" In the current version we
413 * will use "prefetch hint,128(a0)", so "t0-160" is the limit.
414 */
415 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
416 PTR_ADDU t0,a0,a2 /* t0 is the "past the end" address */
417 PTR_SUBU t9,t0,PREFETCH_LIMIT /* t9 is the "last safe pref" address */
418 #endif
419 PREFETCH_FOR_LOAD (0, a1)
420 PREFETCH_FOR_LOAD (1, a1)
421 PREFETCH_FOR_LOAD (2, a1)
422 PREFETCH_FOR_LOAD (3, a1)
423 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
424 PREFETCH_FOR_STORE (1, a0)
425 PREFETCH_FOR_STORE (2, a0)
426 PREFETCH_FOR_STORE (3, a0)
427 #endif
428 #if defined(RETURN_FIRST_PREFETCH) && defined(USE_PREFETCH)
429 # if PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE
430 sltu v1,t9,a0
431 bgtz v1,L(skip_set)
432 nop
433 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*4)
434 L(skip_set):
435 # else
436 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*1)
437 # endif
438 #endif
439 #if defined(RETURN_LAST_PREFETCH) && defined(USE_PREFETCH) \
440 && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
441 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*3)
442 # ifdef USE_DOUBLE
443 PTR_ADDIU v0,v0,32
444 # endif
445 #endif
446 L(loop16w):
447 C_LD t0,UNIT(0)(a1)
448 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
449 sltu v1,t9,a0 /* If a0 > t9 don't use next prefetch */
450 bgtz v1,L(skip_pref)
451 #endif
452 C_LD t1,UNIT(1)(a1)
453 #ifdef R6_CODE
454 PREFETCH_FOR_STORE (2, a0)
455 #else
456 PREFETCH_FOR_STORE (4, a0)
457 PREFETCH_FOR_STORE (5, a0)
458 #endif
459 #if defined(RETURN_LAST_PREFETCH) && defined(USE_PREFETCH)
460 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*5)
461 # ifdef USE_DOUBLE
462 PTR_ADDIU v0,v0,32
463 # endif
464 #endif
465 L(skip_pref):
466 C_LD REG2,UNIT(2)(a1)
467 C_LD REG3,UNIT(3)(a1)
468 C_LD REG4,UNIT(4)(a1)
469 C_LD REG5,UNIT(5)(a1)
470 C_LD REG6,UNIT(6)(a1)
471 C_LD REG7,UNIT(7)(a1)
472 #ifdef R6_CODE
473 PREFETCH_FOR_LOAD (3, a1)
474 #else
475 PREFETCH_FOR_LOAD (4, a1)
476 #endif
477 C_ST t0,UNIT(0)(a0)
478 C_ST t1,UNIT(1)(a0)
479 C_ST REG2,UNIT(2)(a0)
480 C_ST REG3,UNIT(3)(a0)
481 C_ST REG4,UNIT(4)(a0)
482 C_ST REG5,UNIT(5)(a0)
483 C_ST REG6,UNIT(6)(a0)
484 C_ST REG7,UNIT(7)(a0)
486 C_LD t0,UNIT(8)(a1)
487 C_LD t1,UNIT(9)(a1)
488 C_LD REG2,UNIT(10)(a1)
489 C_LD REG3,UNIT(11)(a1)
490 C_LD REG4,UNIT(12)(a1)
491 C_LD REG5,UNIT(13)(a1)
492 C_LD REG6,UNIT(14)(a1)
493 C_LD REG7,UNIT(15)(a1)
494 #ifndef R6_CODE
495 PREFETCH_FOR_LOAD (5, a1)
496 #endif
497 C_ST t0,UNIT(8)(a0)
498 C_ST t1,UNIT(9)(a0)
499 C_ST REG2,UNIT(10)(a0)
500 C_ST REG3,UNIT(11)(a0)
501 C_ST REG4,UNIT(12)(a0)
502 C_ST REG5,UNIT(13)(a0)
503 C_ST REG6,UNIT(14)(a0)
504 C_ST REG7,UNIT(15)(a0)
505 PTR_ADDIU a0,a0,UNIT(16) /* adding 64/128 to dest */
506 bne a0,a3,L(loop16w)
507 PTR_ADDIU a1,a1,UNIT(16) /* adding 64/128 to src */
508 move a2,t8
510 /* Here we have src and dest word-aligned but less than 64-bytes or
511 * 128 bytes to go. Check for a 32(64) byte chunk and copy if if there
512 * is one. Otherwise jump down to L(chk1w) to handle the tail end of
513 * the copy.
514 */
516 L(chkw):
517 PREFETCH_FOR_LOAD (0, a1)
518 andi t8,a2,NSIZEMASK /* Is there a 32-byte/64-byte chunk. */
519 /* The t8 is the reminder count past 32-bytes */
520 beq a2,t8,L(chk1w) /* When a2=t8, no 32-byte chunk */
521 nop
522 C_LD t0,UNIT(0)(a1)
523 C_LD t1,UNIT(1)(a1)
524 C_LD REG2,UNIT(2)(a1)
525 C_LD REG3,UNIT(3)(a1)
526 C_LD REG4,UNIT(4)(a1)
527 C_LD REG5,UNIT(5)(a1)
528 C_LD REG6,UNIT(6)(a1)
529 C_LD REG7,UNIT(7)(a1)
530 PTR_ADDIU a1,a1,UNIT(8)
531 C_ST t0,UNIT(0)(a0)
532 C_ST t1,UNIT(1)(a0)
533 C_ST REG2,UNIT(2)(a0)
534 C_ST REG3,UNIT(3)(a0)
535 C_ST REG4,UNIT(4)(a0)
536 C_ST REG5,UNIT(5)(a0)
537 C_ST REG6,UNIT(6)(a0)
538 C_ST REG7,UNIT(7)(a0)
539 PTR_ADDIU a0,a0,UNIT(8)
541 /*
542 * Here we have less than 32(64) bytes to copy. Set up for a loop to
543 * copy one word (or double word) at a time. Set a2 to count how many
544 * bytes we have to copy after all the word (or double word) chunks are
545 * copied and a3 to the dst pointer after all the (d)word chunks have
546 * been copied. We will loop, incrementing a0 and a1 until a0 equals a3.
547 */
548 L(chk1w):
549 andi a2,t8,(NSIZE-1) /* a2 is the reminder past one (d)word chunks */
550 beq a2,t8,L(lastb)
551 PTR_SUBU a3,t8,a2 /* a3 is count of bytes in one (d)word chunks */
552 PTR_ADDU a3,a0,a3 /* a3 is the dst address after loop */
554 /* copying in words (4-byte or 8-byte chunks) */
555 L(wordCopy_loop):
556 C_LD REG3,UNIT(0)(a1)
557 PTR_ADDIU a0,a0,UNIT(1)
558 PTR_ADDIU a1,a1,UNIT(1)
559 bne a0,a3,L(wordCopy_loop)
560 C_ST REG3,UNIT(-1)(a0)
562 /* Copy the last 8 (or 16) bytes */
563 L(lastb):
564 blez a2,L(leave)
565 PTR_ADDU a3,a0,a2 /* a3 is the last dst address */
566 L(lastbloop):
567 lb v1,0(a1)
568 PTR_ADDIU a0,a0,1
569 PTR_ADDIU a1,a1,1
570 bne a0,a3,L(lastbloop)
571 sb v1,-1(a0)
572 L(leave):
573 j ra
574 nop
576 #ifndef R6_CODE
577 /*
578 * UNALIGNED case, got here with a3 = "negu a0"
579 * This code is nearly identical to the aligned code above
580 * but only the destination (not the source) gets aligned
581 * so we need to do partial loads of the source followed
582 * by normal stores to the destination (once we have aligned
583 * the destination).
584 */
586 L(unaligned):
587 andi a3,a3,(NSIZE-1) /* copy a3 bytes to align a0/a1 */
588 beqz a3,L(ua_chk16w) /* if a3=0, it is already aligned */
589 PTR_SUBU a2,a2,a3 /* a2 is the remining bytes count */
591 C_LDHI v1,UNIT(0)(a1)
592 C_LDLO v1,UNITM1(1)(a1)
593 PTR_ADDU a1,a1,a3
594 C_STHI v1,UNIT(0)(a0)
595 PTR_ADDU a0,a0,a3
597 /*
598 * Now the destination (but not the source) is aligned
599 * Set a2 to count how many bytes we have to copy after all the 64/128 byte
600 * chunks are copied and a3 to the dst pointer after all the 64/128 byte
601 * chunks have been copied. We will loop, incrementing a0 and a1 until a0
602 * equals a3.
603 */
605 L(ua_chk16w):
606 andi t8,a2,NSIZEDMASK /* any whole 64-byte/128-byte chunks? */
607 beq a2,t8,L(ua_chkw) /* if a2==t8, no 64-byte/128-byte chunks */
608 PTR_SUBU a3,a2,t8 /* subtract from a2 the reminder */
609 PTR_ADDU a3,a0,a3 /* Now a3 is the final dst after loop */
611 # if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
612 PTR_ADDU t0,a0,a2 /* t0 is the "past the end" address */
613 PTR_SUBU t9,t0,PREFETCH_LIMIT /* t9 is the "last safe pref" address */
614 # endif
615 PREFETCH_FOR_LOAD (0, a1)
616 PREFETCH_FOR_LOAD (1, a1)
617 PREFETCH_FOR_LOAD (2, a1)
618 # if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
619 PREFETCH_FOR_STORE (1, a0)
620 PREFETCH_FOR_STORE (2, a0)
621 PREFETCH_FOR_STORE (3, a0)
622 # endif
623 # if defined(RETURN_FIRST_PREFETCH) && defined(USE_PREFETCH)
624 # if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
625 sltu v1,t9,a0
626 bgtz v1,L(ua_skip_set)
627 nop
628 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*4)
629 L(ua_skip_set):
630 # else
631 PTR_ADDIU v0,a0,(PREFETCH_CHUNK*1)
632 # endif
633 # endif
634 L(ua_loop16w):
635 PREFETCH_FOR_LOAD (3, a1)
636 C_LDHI t0,UNIT(0)(a1)
637 C_LDHI t1,UNIT(1)(a1)
638 C_LDHI REG2,UNIT(2)(a1)
639 # if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
640 sltu v1,t9,a0
641 bgtz v1,L(ua_skip_pref)
642 # endif
643 C_LDHI REG3,UNIT(3)(a1)
644 PREFETCH_FOR_STORE (4, a0)
645 PREFETCH_FOR_STORE (5, a0)
646 L(ua_skip_pref):
647 C_LDHI REG4,UNIT(4)(a1)
648 C_LDHI REG5,UNIT(5)(a1)
649 C_LDHI REG6,UNIT(6)(a1)
650 C_LDHI REG7,UNIT(7)(a1)
651 C_LDLO t0,UNITM1(1)(a1)
652 C_LDLO t1,UNITM1(2)(a1)
653 C_LDLO REG2,UNITM1(3)(a1)
654 C_LDLO REG3,UNITM1(4)(a1)
655 C_LDLO REG4,UNITM1(5)(a1)
656 C_LDLO REG5,UNITM1(6)(a1)
657 C_LDLO REG6,UNITM1(7)(a1)
658 C_LDLO REG7,UNITM1(8)(a1)
659 PREFETCH_FOR_LOAD (4, a1)
660 C_ST t0,UNIT(0)(a0)
661 C_ST t1,UNIT(1)(a0)
662 C_ST REG2,UNIT(2)(a0)
663 C_ST REG3,UNIT(3)(a0)
664 C_ST REG4,UNIT(4)(a0)
665 C_ST REG5,UNIT(5)(a0)
666 C_ST REG6,UNIT(6)(a0)
667 C_ST REG7,UNIT(7)(a0)
668 C_LDHI t0,UNIT(8)(a1)
669 C_LDHI t1,UNIT(9)(a1)
670 C_LDHI REG2,UNIT(10)(a1)
671 C_LDHI REG3,UNIT(11)(a1)
672 C_LDHI REG4,UNIT(12)(a1)
673 C_LDHI REG5,UNIT(13)(a1)
674 C_LDHI REG6,UNIT(14)(a1)
675 C_LDHI REG7,UNIT(15)(a1)
676 C_LDLO t0,UNITM1(9)(a1)
677 C_LDLO t1,UNITM1(10)(a1)
678 C_LDLO REG2,UNITM1(11)(a1)
679 C_LDLO REG3,UNITM1(12)(a1)
680 C_LDLO REG4,UNITM1(13)(a1)
681 C_LDLO REG5,UNITM1(14)(a1)
682 C_LDLO REG6,UNITM1(15)(a1)
683 C_LDLO REG7,UNITM1(16)(a1)
684 PREFETCH_FOR_LOAD (5, a1)
685 C_ST t0,UNIT(8)(a0)
686 C_ST t1,UNIT(9)(a0)
687 C_ST REG2,UNIT(10)(a0)
688 C_ST REG3,UNIT(11)(a0)
689 C_ST REG4,UNIT(12)(a0)
690 C_ST REG5,UNIT(13)(a0)
691 C_ST REG6,UNIT(14)(a0)
692 C_ST REG7,UNIT(15)(a0)
693 PTR_ADDIU a0,a0,UNIT(16) /* adding 64/128 to dest */
694 bne a0,a3,L(ua_loop16w)
695 PTR_ADDIU a1,a1,UNIT(16) /* adding 64/128 to src */
696 move a2,t8
698 /* Here we have src and dest word-aligned but less than 64-bytes or
699 * 128 bytes to go. Check for a 32(64) byte chunk and copy if if there
700 * is one. Otherwise jump down to L(ua_chk1w) to handle the tail end of
701 * the copy. */
703 L(ua_chkw):
704 PREFETCH_FOR_LOAD (0, a1)
705 andi t8,a2,NSIZEMASK /* Is there a 32-byte/64-byte chunk. */
706 /* t8 is the reminder count past 32-bytes */
707 beq a2,t8,L(ua_chk1w) /* When a2=t8, no 32-byte chunk */
708 nop
709 C_LDHI t0,UNIT(0)(a1)
710 C_LDHI t1,UNIT(1)(a1)
711 C_LDHI REG2,UNIT(2)(a1)
712 C_LDHI REG3,UNIT(3)(a1)
713 C_LDHI REG4,UNIT(4)(a1)
714 C_LDHI REG5,UNIT(5)(a1)
715 C_LDHI REG6,UNIT(6)(a1)
716 C_LDHI REG7,UNIT(7)(a1)
717 C_LDLO t0,UNITM1(1)(a1)
718 C_LDLO t1,UNITM1(2)(a1)
719 C_LDLO REG2,UNITM1(3)(a1)
720 C_LDLO REG3,UNITM1(4)(a1)
721 C_LDLO REG4,UNITM1(5)(a1)
722 C_LDLO REG5,UNITM1(6)(a1)
723 C_LDLO REG6,UNITM1(7)(a1)
724 C_LDLO REG7,UNITM1(8)(a1)
725 PTR_ADDIU a1,a1,UNIT(8)
726 C_ST t0,UNIT(0)(a0)
727 C_ST t1,UNIT(1)(a0)
728 C_ST REG2,UNIT(2)(a0)
729 C_ST REG3,UNIT(3)(a0)
730 C_ST REG4,UNIT(4)(a0)
731 C_ST REG5,UNIT(5)(a0)
732 C_ST REG6,UNIT(6)(a0)
733 C_ST REG7,UNIT(7)(a0)
734 PTR_ADDIU a0,a0,UNIT(8)
735 /*
736 * Here we have less than 32(64) bytes to copy. Set up for a loop to
737 * copy one word (or double word) at a time.
738 */
739 L(ua_chk1w):
740 andi a2,t8,(NSIZE-1) /* a2 is the reminder past one (d)word chunks */
741 beq a2,t8,L(ua_smallCopy)
742 PTR_SUBU a3,t8,a2 /* a3 is count of bytes in one (d)word chunks */
743 PTR_ADDU a3,a0,a3 /* a3 is the dst address after loop */
745 /* copying in words (4-byte or 8-byte chunks) */
746 L(ua_wordCopy_loop):
747 C_LDHI v1,UNIT(0)(a1)
748 C_LDLO v1,UNITM1(1)(a1)
749 PTR_ADDIU a0,a0,UNIT(1)
750 PTR_ADDIU a1,a1,UNIT(1)
751 bne a0,a3,L(ua_wordCopy_loop)
752 C_ST v1,UNIT(-1)(a0)
754 /* Copy the last 8 (or 16) bytes */
755 L(ua_smallCopy):
756 beqz a2,L(leave)
757 PTR_ADDU a3,a0,a2 /* a3 is the last dst address */
758 L(ua_smallCopy_loop):
759 lb v1,0(a1)
760 PTR_ADDIU a0,a0,1
761 PTR_ADDIU a1,a1,1
762 bne a0,a3,L(ua_smallCopy_loop)
763 sb v1,-1(a0)
765 j ra
766 nop
768 #else /* R6_CODE */
770 # if __MIPSEB
771 # define SWAP_REGS(X,Y) X, Y
772 # define ALIGN_OFFSET(N) (N)
773 # else
774 # define SWAP_REGS(X,Y) Y, X
775 # define ALIGN_OFFSET(N) (NSIZE-N)
776 # endif
777 # define R6_UNALIGNED_WORD_COPY(BYTEOFFSET) \
778 andi REG7, a2, (NSIZE-1);/* REG7 is # of bytes to by bytes. */ \
779 beq REG7, a2, L(lastb); /* Check for bytes to copy by word */ \
780 PTR_SUBU a3, a2, REG7; /* a3 is number of bytes to be copied in */ \
781 /* (d)word chunks. */ \
782 move a2, REG7; /* a2 is # of bytes to copy byte by byte */ \
783 /* after word loop is finished. */ \
784 PTR_ADDU REG6, a0, a3; /* REG6 is the dst address after loop. */ \
785 PTR_SUBU REG2, a1, t8; /* REG2 is the aligned src address. */ \
786 PTR_ADDU a1, a1, a3; /* a1 is addr of source after word loop. */ \
787 C_LD t0, UNIT(0)(REG2); /* Load first part of source. */ \
788 L(r6_ua_wordcopy##BYTEOFFSET): \
789 C_LD t1, UNIT(1)(REG2); /* Load second part of source. */ \
790 C_ALIGN REG3, SWAP_REGS(t1,t0), ALIGN_OFFSET(BYTEOFFSET); \
791 PTR_ADDIU a0, a0, UNIT(1); /* Increment destination pointer. */ \
792 PTR_ADDIU REG2, REG2, UNIT(1); /* Increment aligned source pointer.*/ \
793 move t0, t1; /* Move second part of source to first. */ \
794 bne a0, REG6,L(r6_ua_wordcopy##BYTEOFFSET); \
795 C_ST REG3, UNIT(-1)(a0); \
796 j L(lastb); \
797 nop
799 /* We are generating R6 code, the destination is 4 byte aligned and
800 the source is not 4 byte aligned. t8 is 1, 2, or 3 depending on the
801 alignment of the source. */
803 L(r6_unaligned1):
804 R6_UNALIGNED_WORD_COPY(1)
805 L(r6_unaligned2):
806 R6_UNALIGNED_WORD_COPY(2)
807 L(r6_unaligned3):
808 R6_UNALIGNED_WORD_COPY(3)
809 # ifdef USE_DOUBLE
810 L(r6_unaligned4):
811 R6_UNALIGNED_WORD_COPY(4)
812 L(r6_unaligned5):
813 R6_UNALIGNED_WORD_COPY(5)
814 L(r6_unaligned6):
815 R6_UNALIGNED_WORD_COPY(6)
816 L(r6_unaligned7):
817 R6_UNALIGNED_WORD_COPY(7)
818 # endif
819 #endif /* R6_CODE */
821 .set at
822 .set reorder
823 END(MEMCPY_NAME)
824 #ifndef ANDROID_CHANGES
825 # ifdef _LIBC
826 libc_hidden_builtin_def (MEMCPY_NAME)
827 # endif
828 #endif