string/wordcopy.c

   1 /* _memcopy.c -- subroutines for memory copy functions.
   2    Copyright (C) 1991-2023 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 /* BE VERY CAREFUL IF YOU CHANGE THIS CODE...!  */
  20
  21 #include <stddef.h>
  22 #include <libc-diag.h>
  23 /* Compiling with -O1 might warn that 'a2' and 'a3' may be used
  24    uninitialized.  There are only two ways to arrive at labels 'do4', 'do3'
  25    or 'do1', all of which use 'a2' or 'a3' in the MERGE macro: either from
  26    the earlier switch case statement or via a loop iteration.  In all cases
  27    the switch statement or previous loop sets both 'a2' and 'a3'.
  28
  29    Since the usage is within the MERGE macro we disable the
  30    warning in the definition, but only in this file.  */
  31 DIAG_PUSH_NEEDS_COMMENT;
  32 DIAG_IGNORE_NEEDS_COMMENT (6, "-Wmaybe-uninitialized");
  33 #include <memcopy.h>
  34 DIAG_POP_NEEDS_COMMENT;
  35
  36 /* _wordcopy_fwd_aligned -- Copy block beginning at SRCP to
  37    block beginning at DSTP with LEN `op_t' words (not LEN bytes!).
  38    Both SRCP and DSTP should be aligned for memory operations on `op_t's.  */
  39
  40 #ifndef WORDCOPY_FWD_ALIGNED
  41 # define WORDCOPY_FWD_ALIGNED _wordcopy_fwd_aligned
  42 #endif
  43
  44 void
  45 WORDCOPY_FWD_ALIGNED (long int dstp, long int srcp, size_t len)
  46 {
  47   op_t a0, a1;
  48
  49   switch (len % 8)
  50     {
  51     case 2:
  52       a0 = ((op_t *) srcp)[0];
  53       srcp -= 6 * OPSIZ;
  54       dstp -= 7 * OPSIZ;
  55       len += 6;
  56       goto do1;
  57     case 3:
  58       a1 = ((op_t *) srcp)[0];
  59       srcp -= 5 * OPSIZ;
  60       dstp -= 6 * OPSIZ;
  61       len += 5;
  62       goto do2;
  63     case 4:
  64       a0 = ((op_t *) srcp)[0];
  65       srcp -= 4 * OPSIZ;
  66       dstp -= 5 * OPSIZ;
  67       len += 4;
  68       goto do3;
  69     case 5:
  70       a1 = ((op_t *) srcp)[0];
  71       srcp -= 3 * OPSIZ;
  72       dstp -= 4 * OPSIZ;
  73       len += 3;
  74       goto do4;
  75     case 6:
  76       a0 = ((op_t *) srcp)[0];
  77       srcp -= 2 * OPSIZ;
  78       dstp -= 3 * OPSIZ;
  79       len += 2;
  80       goto do5;
  81     case 7:
  82       a1 = ((op_t *) srcp)[0];
  83       srcp -= 1 * OPSIZ;
  84       dstp -= 2 * OPSIZ;
  85       len += 1;
  86       goto do6;
  87
  88     case 0:
  89       if (OP_T_THRES <= 3 * OPSIZ && len == 0)
  90         return;
  91       a0 = ((op_t *) srcp)[0];
  92       srcp -= 0 * OPSIZ;
  93       dstp -= 1 * OPSIZ;
  94       goto do7;
  95     case 1:
  96       a1 = ((op_t *) srcp)[0];
  97       srcp -=-1 * OPSIZ;
  98       dstp -= 0 * OPSIZ;
  99       len -= 1;
 100       if (OP_T_THRES <= 3 * OPSIZ && len == 0)
 101         goto do0;
 102       goto do8;                 /* No-op.  */
 103     }
 104
 105   do
 106     {
 107     do8:
 108       a0 = ((op_t *) srcp)[0];
 109       /* Compiling with -O1 may warn that 'a1' may be used uninitialized.
 110          There are only two ways to arrive at label 'do8' and they are via a
 111          do-while loop iteration or directly via the earlier switch 'case 1:'
 112          case. The switch case always sets 'a1' and all previous loop
 113          iterations will also have set 'a1' before the use.  */
 114       DIAG_PUSH_NEEDS_COMMENT;
 115       DIAG_IGNORE_NEEDS_COMMENT (6, "-Wmaybe-uninitialized");
 116       ((op_t *) dstp)[0] = a1;
 117       DIAG_POP_NEEDS_COMMENT;
 118     do7:
 119       a1 = ((op_t *) srcp)[1];
 120       ((op_t *) dstp)[1] = a0;
 121     do6:
 122       a0 = ((op_t *) srcp)[2];
 123       ((op_t *) dstp)[2] = a1;
 124     do5:
 125       a1 = ((op_t *) srcp)[3];
 126       ((op_t *) dstp)[3] = a0;
 127     do4:
 128       a0 = ((op_t *) srcp)[4];
 129       ((op_t *) dstp)[4] = a1;
 130     do3:
 131       a1 = ((op_t *) srcp)[5];
 132       ((op_t *) dstp)[5] = a0;
 133     do2:
 134       a0 = ((op_t *) srcp)[6];
 135       ((op_t *) dstp)[6] = a1;
 136     do1:
 137       a1 = ((op_t *) srcp)[7];
 138       ((op_t *) dstp)[7] = a0;
 139
 140       srcp += 8 * OPSIZ;
 141       dstp += 8 * OPSIZ;
 142       len -= 8;
 143     }
 144   while (len != 0);
 145
 146   /* This is the right position for do0.  Please don't move
 147      it into the loop.  */
 148  do0:
 149   ((op_t *) dstp)[0] = a1;
 150 }
 151
 152 /* _wordcopy_fwd_dest_aligned -- Copy block beginning at SRCP to
 153    block beginning at DSTP with LEN `op_t' words (not LEN bytes!).
 154    DSTP should be aligned for memory operations on `op_t's, but SRCP must
 155    *not* be aligned.  */
 156
 157 #ifndef WORDCOPY_FWD_DEST_ALIGNED
 158 # define WORDCOPY_FWD_DEST_ALIGNED _wordcopy_fwd_dest_aligned
 159 #endif
 160
 161 void
 162 WORDCOPY_FWD_DEST_ALIGNED (long int dstp, long int srcp, size_t len)
 163 {
 164   op_t a0, a1, a2, a3;
 165   int sh_1, sh_2;
 166
 167   /* Calculate how to shift a word read at the memory operation
 168      aligned srcp to make it aligned for copy.  */
 169
 170   sh_1 = 8 * (srcp % OPSIZ);
 171   sh_2 = 8 * OPSIZ - sh_1;
 172
 173   /* Make SRCP aligned by rounding it down to the beginning of the `op_t'
 174      it points in the middle of.  */
 175   srcp &= -OPSIZ;
 176
 177   switch (len % 4)
 178     {
 179     case 2:
 180       a1 = ((op_t *) srcp)[0];
 181       a2 = ((op_t *) srcp)[1];
 182       srcp -= 1 * OPSIZ;
 183       dstp -= 3 * OPSIZ;
 184       len += 2;
 185       goto do1;
 186     case 3:
 187       a0 = ((op_t *) srcp)[0];
 188       a1 = ((op_t *) srcp)[1];
 189       srcp -= 0 * OPSIZ;
 190       dstp -= 2 * OPSIZ;
 191       len += 1;
 192       goto do2;
 193     case 0:
 194       if (OP_T_THRES <= 3 * OPSIZ && len == 0)
 195         return;
 196       a3 = ((op_t *) srcp)[0];
 197       a0 = ((op_t *) srcp)[1];
 198       srcp -=-1 * OPSIZ;
 199       dstp -= 1 * OPSIZ;
 200       len += 0;
 201       goto do3;
 202     case 1:
 203       a2 = ((op_t *) srcp)[0];
 204       a3 = ((op_t *) srcp)[1];
 205       srcp -=-2 * OPSIZ;
 206       dstp -= 0 * OPSIZ;
 207       len -= 1;
 208       if (OP_T_THRES <= 3 * OPSIZ && len == 0)
 209         goto do0;
 210       goto do4;                 /* No-op.  */
 211     }
 212
 213   do
 214     {
 215     do4:
 216       a0 = ((op_t *) srcp)[0];
 217       ((op_t *) dstp)[0] = MERGE (a2, sh_1, a3, sh_2);
 218     do3:
 219       a1 = ((op_t *) srcp)[1];
 220       ((op_t *) dstp)[1] = MERGE (a3, sh_1, a0, sh_2);
 221     do2:
 222       a2 = ((op_t *) srcp)[2];
 223       ((op_t *) dstp)[2] = MERGE (a0, sh_1, a1, sh_2);
 224     do1:
 225       a3 = ((op_t *) srcp)[3];
 226       ((op_t *) dstp)[3] = MERGE (a1, sh_1, a2, sh_2);
 227
 228       srcp += 4 * OPSIZ;
 229       dstp += 4 * OPSIZ;
 230       len -= 4;
 231     }
 232   while (len != 0);
 233
 234   /* This is the right position for do0.  Please don't move
 235      it into the loop.  */
 236  do0:
 237   ((op_t *) dstp)[0] = MERGE (a2, sh_1, a3, sh_2);
 238 }
 239
 240 /* _wordcopy_bwd_aligned -- Copy block finishing right before
 241    SRCP to block finishing right before DSTP with LEN `op_t' words
 242    (not LEN bytes!).  Both SRCP and DSTP should be aligned for memory
 243    operations on `op_t's.  */
 244
 245 #ifndef WORDCOPY_BWD_ALIGNED
 246 # define WORDCOPY_BWD_ALIGNED _wordcopy_bwd_aligned
 247 #endif
 248
 249 void
 250 WORDCOPY_BWD_ALIGNED (long int dstp, long int srcp, size_t len)
 251 {
 252   op_t a0, a1;
 253
 254   switch (len % 8)
 255     {
 256     case 2:
 257       srcp -= 2 * OPSIZ;
 258       dstp -= 1 * OPSIZ;
 259       a0 = ((op_t *) srcp)[1];
 260       len += 6;
 261       goto do1;
 262     case 3:
 263       srcp -= 3 * OPSIZ;
 264       dstp -= 2 * OPSIZ;
 265       a1 = ((op_t *) srcp)[2];
 266       len += 5;
 267       goto do2;
 268     case 4:
 269       srcp -= 4 * OPSIZ;
 270       dstp -= 3 * OPSIZ;
 271       a0 = ((op_t *) srcp)[3];
 272       len += 4;
 273       goto do3;
 274     case 5:
 275       srcp -= 5 * OPSIZ;
 276       dstp -= 4 * OPSIZ;
 277       a1 = ((op_t *) srcp)[4];
 278       len += 3;
 279       goto do4;
 280     case 6:
 281       srcp -= 6 * OPSIZ;
 282       dstp -= 5 * OPSIZ;
 283       a0 = ((op_t *) srcp)[5];
 284       len += 2;
 285       goto do5;
 286     case 7:
 287       srcp -= 7 * OPSIZ;
 288       dstp -= 6 * OPSIZ;
 289       a1 = ((op_t *) srcp)[6];
 290       len += 1;
 291       goto do6;
 292
 293     case 0:
 294       if (OP_T_THRES <= 3 * OPSIZ && len == 0)
 295         return;
 296       srcp -= 8 * OPSIZ;
 297       dstp -= 7 * OPSIZ;
 298       a0 = ((op_t *) srcp)[7];
 299       goto do7;
 300     case 1:
 301       srcp -= 9 * OPSIZ;
 302       dstp -= 8 * OPSIZ;
 303       a1 = ((op_t *) srcp)[8];
 304       len -= 1;
 305       if (OP_T_THRES <= 3 * OPSIZ && len == 0)
 306         goto do0;
 307       goto do8;                 /* No-op.  */
 308     }
 309
 310   do
 311     {
 312     do8:
 313       a0 = ((op_t *) srcp)[7];
 314       /* Check the comment on WORDCOPY_FWD_ALIGNED.  */
 315       DIAG_PUSH_NEEDS_COMMENT;
 316       DIAG_IGNORE_NEEDS_COMMENT (6, "-Wmaybe-uninitialized");
 317       ((op_t *) dstp)[7] = a1;
 318       DIAG_POP_NEEDS_COMMENT;
 319     do7:
 320       a1 = ((op_t *) srcp)[6];
 321       ((op_t *) dstp)[6] = a0;
 322     do6:
 323       a0 = ((op_t *) srcp)[5];
 324       ((op_t *) dstp)[5] = a1;
 325     do5:
 326       a1 = ((op_t *) srcp)[4];
 327       ((op_t *) dstp)[4] = a0;
 328     do4:
 329       a0 = ((op_t *) srcp)[3];
 330       ((op_t *) dstp)[3] = a1;
 331     do3:
 332       a1 = ((op_t *) srcp)[2];
 333       ((op_t *) dstp)[2] = a0;
 334     do2:
 335       a0 = ((op_t *) srcp)[1];
 336       ((op_t *) dstp)[1] = a1;
 337     do1:
 338       a1 = ((op_t *) srcp)[0];
 339       ((op_t *) dstp)[0] = a0;
 340
 341       srcp -= 8 * OPSIZ;
 342       dstp -= 8 * OPSIZ;
 343       len -= 8;
 344     }
 345   while (len != 0);
 346
 347   /* This is the right position for do0.  Please don't move
 348      it into the loop.  */
 349  do0:
 350   ((op_t *) dstp)[7] = a1;
 351 }
 352
 353 /* _wordcopy_bwd_dest_aligned -- Copy block finishing right
 354    before SRCP to block finishing right before DSTP with LEN `op_t'
 355    words (not LEN bytes!).  DSTP should be aligned for memory
 356    operations on `op_t', but SRCP must *not* be aligned.  */
 357
 358 #ifndef WORDCOPY_BWD_DEST_ALIGNED
 359 # define WORDCOPY_BWD_DEST_ALIGNED _wordcopy_bwd_dest_aligned
 360 #endif
 361
 362 void
 363 WORDCOPY_BWD_DEST_ALIGNED (long int dstp, long int srcp, size_t len)
 364 {
 365   op_t a0, a1, a2, a3;
 366   int sh_1, sh_2;
 367
 368   /* Calculate how to shift a word read at the memory operation
 369      aligned srcp to make it aligned for copy.  */
 370
 371   sh_1 = 8 * (srcp % OPSIZ);
 372   sh_2 = 8 * OPSIZ - sh_1;
 373
 374   /* Make srcp aligned by rounding it down to the beginning of the op_t
 375      it points in the middle of.  */
 376   srcp &= -OPSIZ;
 377   srcp += OPSIZ;
 378
 379   switch (len % 4)
 380     {
 381     case 2:
 382       srcp -= 3 * OPSIZ;
 383       dstp -= 1 * OPSIZ;
 384       a2 = ((op_t *) srcp)[2];
 385       a1 = ((op_t *) srcp)[1];
 386       len += 2;
 387       goto do1;
 388     case 3:
 389       srcp -= 4 * OPSIZ;
 390       dstp -= 2 * OPSIZ;
 391       a3 = ((op_t *) srcp)[3];
 392       a2 = ((op_t *) srcp)[2];
 393       len += 1;
 394       goto do2;
 395     case 0:
 396       if (OP_T_THRES <= 3 * OPSIZ && len == 0)
 397         return;
 398       srcp -= 5 * OPSIZ;
 399       dstp -= 3 * OPSIZ;
 400       a0 = ((op_t *) srcp)[4];
 401       a3 = ((op_t *) srcp)[3];
 402       goto do3;
 403     case 1:
 404       srcp -= 6 * OPSIZ;
 405       dstp -= 4 * OPSIZ;
 406       a1 = ((op_t *) srcp)[5];
 407       a0 = ((op_t *) srcp)[4];
 408       len -= 1;
 409       if (OP_T_THRES <= 3 * OPSIZ && len == 0)
 410         goto do0;
 411       goto do4;                 /* No-op.  */
 412     }
 413
 414   do
 415     {
 416     do4:
 417       a3 = ((op_t *) srcp)[3];
 418       ((op_t *) dstp)[3] = MERGE (a0, sh_1, a1, sh_2);
 419     do3:
 420       a2 = ((op_t *) srcp)[2];
 421       ((op_t *) dstp)[2] = MERGE (a3, sh_1, a0, sh_2);
 422     do2:
 423       a1 = ((op_t *) srcp)[1];
 424       ((op_t *) dstp)[1] = MERGE (a2, sh_1, a3, sh_2);
 425     do1:
 426       a0 = ((op_t *) srcp)[0];
 427       ((op_t *) dstp)[0] = MERGE (a1, sh_1, a2, sh_2);
 428
 429       srcp -= 4 * OPSIZ;
 430       dstp -= 4 * OPSIZ;
 431       len -= 4;
 432     }
 433   while (len != 0);
 434
 435   /* This is the right position for do0.  Please don't move
 436      it into the loop.  */
 437  do0:
 438   ((op_t *) dstp)[3] = MERGE (a0, sh_1, a1, sh_2);
 439 }