2 * Generic vectorized operation runtime
4 * Copyright (c) 2018 Linaro
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/host-utils.h"
23 #include "exec/helper-proto.h"
24 #include "tcg/tcg-gvec-desc.h"
27 static inline void clear_high(void *d
, intptr_t oprsz
, uint32_t desc
)
29 intptr_t maxsz
= simd_maxsz(desc
);
32 if (unlikely(maxsz
> oprsz
)) {
33 for (i
= oprsz
; i
< maxsz
; i
+= sizeof(uint64_t)) {
34 *(uint64_t *)(d
+ i
) = 0;
39 void HELPER(gvec_add8
)(void *d
, void *a
, void *b
, uint32_t desc
)
41 intptr_t oprsz
= simd_oprsz(desc
);
44 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
45 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) + *(uint8_t *)(b
+ i
);
47 clear_high(d
, oprsz
, desc
);
50 void HELPER(gvec_add16
)(void *d
, void *a
, void *b
, uint32_t desc
)
52 intptr_t oprsz
= simd_oprsz(desc
);
55 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
56 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) + *(uint16_t *)(b
+ i
);
58 clear_high(d
, oprsz
, desc
);
61 void HELPER(gvec_add32
)(void *d
, void *a
, void *b
, uint32_t desc
)
63 intptr_t oprsz
= simd_oprsz(desc
);
66 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
67 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) + *(uint32_t *)(b
+ i
);
69 clear_high(d
, oprsz
, desc
);
72 void HELPER(gvec_add64
)(void *d
, void *a
, void *b
, uint32_t desc
)
74 intptr_t oprsz
= simd_oprsz(desc
);
77 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
78 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) + *(uint64_t *)(b
+ i
);
80 clear_high(d
, oprsz
, desc
);
83 void HELPER(gvec_adds8
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
85 intptr_t oprsz
= simd_oprsz(desc
);
88 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
89 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) + (uint8_t)b
;
91 clear_high(d
, oprsz
, desc
);
94 void HELPER(gvec_adds16
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
96 intptr_t oprsz
= simd_oprsz(desc
);
99 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
100 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) + (uint16_t)b
;
102 clear_high(d
, oprsz
, desc
);
105 void HELPER(gvec_adds32
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
107 intptr_t oprsz
= simd_oprsz(desc
);
110 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
111 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) + (uint32_t)b
;
113 clear_high(d
, oprsz
, desc
);
116 void HELPER(gvec_adds64
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
118 intptr_t oprsz
= simd_oprsz(desc
);
121 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
122 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) + b
;
124 clear_high(d
, oprsz
, desc
);
127 void HELPER(gvec_sub8
)(void *d
, void *a
, void *b
, uint32_t desc
)
129 intptr_t oprsz
= simd_oprsz(desc
);
132 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
133 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) - *(uint8_t *)(b
+ i
);
135 clear_high(d
, oprsz
, desc
);
138 void HELPER(gvec_sub16
)(void *d
, void *a
, void *b
, uint32_t desc
)
140 intptr_t oprsz
= simd_oprsz(desc
);
143 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
144 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) - *(uint16_t *)(b
+ i
);
146 clear_high(d
, oprsz
, desc
);
149 void HELPER(gvec_sub32
)(void *d
, void *a
, void *b
, uint32_t desc
)
151 intptr_t oprsz
= simd_oprsz(desc
);
154 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
155 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) - *(uint32_t *)(b
+ i
);
157 clear_high(d
, oprsz
, desc
);
160 void HELPER(gvec_sub64
)(void *d
, void *a
, void *b
, uint32_t desc
)
162 intptr_t oprsz
= simd_oprsz(desc
);
165 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
166 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) - *(uint64_t *)(b
+ i
);
168 clear_high(d
, oprsz
, desc
);
171 void HELPER(gvec_subs8
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
173 intptr_t oprsz
= simd_oprsz(desc
);
176 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
177 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) - (uint8_t)b
;
179 clear_high(d
, oprsz
, desc
);
182 void HELPER(gvec_subs16
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
184 intptr_t oprsz
= simd_oprsz(desc
);
187 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
188 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) - (uint16_t)b
;
190 clear_high(d
, oprsz
, desc
);
193 void HELPER(gvec_subs32
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
195 intptr_t oprsz
= simd_oprsz(desc
);
198 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
199 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) - (uint32_t)b
;
201 clear_high(d
, oprsz
, desc
);
204 void HELPER(gvec_subs64
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
206 intptr_t oprsz
= simd_oprsz(desc
);
209 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
210 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) - b
;
212 clear_high(d
, oprsz
, desc
);
215 void HELPER(gvec_mul8
)(void *d
, void *a
, void *b
, uint32_t desc
)
217 intptr_t oprsz
= simd_oprsz(desc
);
220 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
221 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) * *(uint8_t *)(b
+ i
);
223 clear_high(d
, oprsz
, desc
);
226 void HELPER(gvec_mul16
)(void *d
, void *a
, void *b
, uint32_t desc
)
228 intptr_t oprsz
= simd_oprsz(desc
);
231 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
232 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) * *(uint16_t *)(b
+ i
);
234 clear_high(d
, oprsz
, desc
);
237 void HELPER(gvec_mul32
)(void *d
, void *a
, void *b
, uint32_t desc
)
239 intptr_t oprsz
= simd_oprsz(desc
);
242 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
243 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) * *(uint32_t *)(b
+ i
);
245 clear_high(d
, oprsz
, desc
);
248 void HELPER(gvec_mul64
)(void *d
, void *a
, void *b
, uint32_t desc
)
250 intptr_t oprsz
= simd_oprsz(desc
);
253 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
254 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) * *(uint64_t *)(b
+ i
);
256 clear_high(d
, oprsz
, desc
);
259 void HELPER(gvec_muls8
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
261 intptr_t oprsz
= simd_oprsz(desc
);
264 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
265 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) * (uint8_t)b
;
267 clear_high(d
, oprsz
, desc
);
270 void HELPER(gvec_muls16
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
272 intptr_t oprsz
= simd_oprsz(desc
);
275 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
276 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) * (uint16_t)b
;
278 clear_high(d
, oprsz
, desc
);
281 void HELPER(gvec_muls32
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
283 intptr_t oprsz
= simd_oprsz(desc
);
286 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
287 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) * (uint32_t)b
;
289 clear_high(d
, oprsz
, desc
);
292 void HELPER(gvec_muls64
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
294 intptr_t oprsz
= simd_oprsz(desc
);
297 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
298 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) * b
;
300 clear_high(d
, oprsz
, desc
);
303 void HELPER(gvec_neg8
)(void *d
, void *a
, uint32_t desc
)
305 intptr_t oprsz
= simd_oprsz(desc
);
308 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
309 *(uint8_t *)(d
+ i
) = -*(uint8_t *)(a
+ i
);
311 clear_high(d
, oprsz
, desc
);
314 void HELPER(gvec_neg16
)(void *d
, void *a
, uint32_t desc
)
316 intptr_t oprsz
= simd_oprsz(desc
);
319 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
320 *(uint16_t *)(d
+ i
) = -*(uint16_t *)(a
+ i
);
322 clear_high(d
, oprsz
, desc
);
325 void HELPER(gvec_neg32
)(void *d
, void *a
, uint32_t desc
)
327 intptr_t oprsz
= simd_oprsz(desc
);
330 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
331 *(uint32_t *)(d
+ i
) = -*(uint32_t *)(a
+ i
);
333 clear_high(d
, oprsz
, desc
);
336 void HELPER(gvec_neg64
)(void *d
, void *a
, uint32_t desc
)
338 intptr_t oprsz
= simd_oprsz(desc
);
341 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
342 *(uint64_t *)(d
+ i
) = -*(uint64_t *)(a
+ i
);
344 clear_high(d
, oprsz
, desc
);
347 void HELPER(gvec_abs8
)(void *d
, void *a
, uint32_t desc
)
349 intptr_t oprsz
= simd_oprsz(desc
);
352 for (i
= 0; i
< oprsz
; i
+= sizeof(int8_t)) {
353 int8_t aa
= *(int8_t *)(a
+ i
);
354 *(int8_t *)(d
+ i
) = aa
< 0 ? -aa
: aa
;
356 clear_high(d
, oprsz
, desc
);
359 void HELPER(gvec_abs16
)(void *d
, void *a
, uint32_t desc
)
361 intptr_t oprsz
= simd_oprsz(desc
);
364 for (i
= 0; i
< oprsz
; i
+= sizeof(int16_t)) {
365 int16_t aa
= *(int16_t *)(a
+ i
);
366 *(int16_t *)(d
+ i
) = aa
< 0 ? -aa
: aa
;
368 clear_high(d
, oprsz
, desc
);
371 void HELPER(gvec_abs32
)(void *d
, void *a
, uint32_t desc
)
373 intptr_t oprsz
= simd_oprsz(desc
);
376 for (i
= 0; i
< oprsz
; i
+= sizeof(int32_t)) {
377 int32_t aa
= *(int32_t *)(a
+ i
);
378 *(int32_t *)(d
+ i
) = aa
< 0 ? -aa
: aa
;
380 clear_high(d
, oprsz
, desc
);
383 void HELPER(gvec_abs64
)(void *d
, void *a
, uint32_t desc
)
385 intptr_t oprsz
= simd_oprsz(desc
);
388 for (i
= 0; i
< oprsz
; i
+= sizeof(int64_t)) {
389 int64_t aa
= *(int64_t *)(a
+ i
);
390 *(int64_t *)(d
+ i
) = aa
< 0 ? -aa
: aa
;
392 clear_high(d
, oprsz
, desc
);
395 void HELPER(gvec_mov
)(void *d
, void *a
, uint32_t desc
)
397 intptr_t oprsz
= simd_oprsz(desc
);
400 clear_high(d
, oprsz
, desc
);
403 void HELPER(gvec_dup64
)(void *d
, uint32_t desc
, uint64_t c
)
405 intptr_t oprsz
= simd_oprsz(desc
);
411 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
412 *(uint64_t *)(d
+ i
) = c
;
415 clear_high(d
, oprsz
, desc
);
418 void HELPER(gvec_dup32
)(void *d
, uint32_t desc
, uint32_t c
)
420 intptr_t oprsz
= simd_oprsz(desc
);
426 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
427 *(uint32_t *)(d
+ i
) = c
;
430 clear_high(d
, oprsz
, desc
);
433 void HELPER(gvec_dup16
)(void *d
, uint32_t desc
, uint32_t c
)
435 HELPER(gvec_dup32
)(d
, desc
, 0x00010001 * (c
& 0xffff));
438 void HELPER(gvec_dup8
)(void *d
, uint32_t desc
, uint32_t c
)
440 HELPER(gvec_dup32
)(d
, desc
, 0x01010101 * (c
& 0xff));
443 void HELPER(gvec_not
)(void *d
, void *a
, uint32_t desc
)
445 intptr_t oprsz
= simd_oprsz(desc
);
448 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
449 *(uint64_t *)(d
+ i
) = ~*(uint64_t *)(a
+ i
);
451 clear_high(d
, oprsz
, desc
);
454 void HELPER(gvec_and
)(void *d
, void *a
, void *b
, uint32_t desc
)
456 intptr_t oprsz
= simd_oprsz(desc
);
459 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
460 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) & *(uint64_t *)(b
+ i
);
462 clear_high(d
, oprsz
, desc
);
465 void HELPER(gvec_or
)(void *d
, void *a
, void *b
, uint32_t desc
)
467 intptr_t oprsz
= simd_oprsz(desc
);
470 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
471 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) | *(uint64_t *)(b
+ i
);
473 clear_high(d
, oprsz
, desc
);
476 void HELPER(gvec_xor
)(void *d
, void *a
, void *b
, uint32_t desc
)
478 intptr_t oprsz
= simd_oprsz(desc
);
481 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
482 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) ^ *(uint64_t *)(b
+ i
);
484 clear_high(d
, oprsz
, desc
);
487 void HELPER(gvec_andc
)(void *d
, void *a
, void *b
, uint32_t desc
)
489 intptr_t oprsz
= simd_oprsz(desc
);
492 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
493 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) &~ *(uint64_t *)(b
+ i
);
495 clear_high(d
, oprsz
, desc
);
498 void HELPER(gvec_orc
)(void *d
, void *a
, void *b
, uint32_t desc
)
500 intptr_t oprsz
= simd_oprsz(desc
);
503 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
504 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) |~ *(uint64_t *)(b
+ i
);
506 clear_high(d
, oprsz
, desc
);
509 void HELPER(gvec_nand
)(void *d
, void *a
, void *b
, uint32_t desc
)
511 intptr_t oprsz
= simd_oprsz(desc
);
514 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
515 *(uint64_t *)(d
+ i
) = ~(*(uint64_t *)(a
+ i
) & *(uint64_t *)(b
+ i
));
517 clear_high(d
, oprsz
, desc
);
520 void HELPER(gvec_nor
)(void *d
, void *a
, void *b
, uint32_t desc
)
522 intptr_t oprsz
= simd_oprsz(desc
);
525 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
526 *(uint64_t *)(d
+ i
) = ~(*(uint64_t *)(a
+ i
) | *(uint64_t *)(b
+ i
));
528 clear_high(d
, oprsz
, desc
);
531 void HELPER(gvec_eqv
)(void *d
, void *a
, void *b
, uint32_t desc
)
533 intptr_t oprsz
= simd_oprsz(desc
);
536 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
537 *(uint64_t *)(d
+ i
) = ~(*(uint64_t *)(a
+ i
) ^ *(uint64_t *)(b
+ i
));
539 clear_high(d
, oprsz
, desc
);
542 void HELPER(gvec_ands
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
544 intptr_t oprsz
= simd_oprsz(desc
);
547 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
548 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) & b
;
550 clear_high(d
, oprsz
, desc
);
553 void HELPER(gvec_xors
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
555 intptr_t oprsz
= simd_oprsz(desc
);
558 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
559 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) ^ b
;
561 clear_high(d
, oprsz
, desc
);
564 void HELPER(gvec_ors
)(void *d
, void *a
, uint64_t b
, uint32_t desc
)
566 intptr_t oprsz
= simd_oprsz(desc
);
569 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
570 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) | b
;
572 clear_high(d
, oprsz
, desc
);
575 void HELPER(gvec_shl8i
)(void *d
, void *a
, uint32_t desc
)
577 intptr_t oprsz
= simd_oprsz(desc
);
578 int shift
= simd_data(desc
);
581 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
582 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) << shift
;
584 clear_high(d
, oprsz
, desc
);
587 void HELPER(gvec_shl16i
)(void *d
, void *a
, uint32_t desc
)
589 intptr_t oprsz
= simd_oprsz(desc
);
590 int shift
= simd_data(desc
);
593 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
594 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) << shift
;
596 clear_high(d
, oprsz
, desc
);
599 void HELPER(gvec_shl32i
)(void *d
, void *a
, uint32_t desc
)
601 intptr_t oprsz
= simd_oprsz(desc
);
602 int shift
= simd_data(desc
);
605 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
606 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) << shift
;
608 clear_high(d
, oprsz
, desc
);
611 void HELPER(gvec_shl64i
)(void *d
, void *a
, uint32_t desc
)
613 intptr_t oprsz
= simd_oprsz(desc
);
614 int shift
= simd_data(desc
);
617 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
618 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) << shift
;
620 clear_high(d
, oprsz
, desc
);
623 void HELPER(gvec_shr8i
)(void *d
, void *a
, uint32_t desc
)
625 intptr_t oprsz
= simd_oprsz(desc
);
626 int shift
= simd_data(desc
);
629 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
630 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) >> shift
;
632 clear_high(d
, oprsz
, desc
);
635 void HELPER(gvec_shr16i
)(void *d
, void *a
, uint32_t desc
)
637 intptr_t oprsz
= simd_oprsz(desc
);
638 int shift
= simd_data(desc
);
641 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
642 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) >> shift
;
644 clear_high(d
, oprsz
, desc
);
647 void HELPER(gvec_shr32i
)(void *d
, void *a
, uint32_t desc
)
649 intptr_t oprsz
= simd_oprsz(desc
);
650 int shift
= simd_data(desc
);
653 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
654 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) >> shift
;
656 clear_high(d
, oprsz
, desc
);
659 void HELPER(gvec_shr64i
)(void *d
, void *a
, uint32_t desc
)
661 intptr_t oprsz
= simd_oprsz(desc
);
662 int shift
= simd_data(desc
);
665 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
666 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) >> shift
;
668 clear_high(d
, oprsz
, desc
);
671 void HELPER(gvec_sar8i
)(void *d
, void *a
, uint32_t desc
)
673 intptr_t oprsz
= simd_oprsz(desc
);
674 int shift
= simd_data(desc
);
677 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
678 *(int8_t *)(d
+ i
) = *(int8_t *)(a
+ i
) >> shift
;
680 clear_high(d
, oprsz
, desc
);
683 void HELPER(gvec_sar16i
)(void *d
, void *a
, uint32_t desc
)
685 intptr_t oprsz
= simd_oprsz(desc
);
686 int shift
= simd_data(desc
);
689 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
690 *(int16_t *)(d
+ i
) = *(int16_t *)(a
+ i
) >> shift
;
692 clear_high(d
, oprsz
, desc
);
695 void HELPER(gvec_sar32i
)(void *d
, void *a
, uint32_t desc
)
697 intptr_t oprsz
= simd_oprsz(desc
);
698 int shift
= simd_data(desc
);
701 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
702 *(int32_t *)(d
+ i
) = *(int32_t *)(a
+ i
) >> shift
;
704 clear_high(d
, oprsz
, desc
);
707 void HELPER(gvec_sar64i
)(void *d
, void *a
, uint32_t desc
)
709 intptr_t oprsz
= simd_oprsz(desc
);
710 int shift
= simd_data(desc
);
713 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
714 *(int64_t *)(d
+ i
) = *(int64_t *)(a
+ i
) >> shift
;
716 clear_high(d
, oprsz
, desc
);
719 void HELPER(gvec_shl8v
)(void *d
, void *a
, void *b
, uint32_t desc
)
721 intptr_t oprsz
= simd_oprsz(desc
);
724 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
725 uint8_t sh
= *(uint8_t *)(b
+ i
) & 7;
726 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) << sh
;
728 clear_high(d
, oprsz
, desc
);
731 void HELPER(gvec_shl16v
)(void *d
, void *a
, void *b
, uint32_t desc
)
733 intptr_t oprsz
= simd_oprsz(desc
);
736 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
737 uint8_t sh
= *(uint16_t *)(b
+ i
) & 15;
738 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) << sh
;
740 clear_high(d
, oprsz
, desc
);
743 void HELPER(gvec_shl32v
)(void *d
, void *a
, void *b
, uint32_t desc
)
745 intptr_t oprsz
= simd_oprsz(desc
);
748 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
749 uint8_t sh
= *(uint32_t *)(b
+ i
) & 31;
750 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) << sh
;
752 clear_high(d
, oprsz
, desc
);
755 void HELPER(gvec_shl64v
)(void *d
, void *a
, void *b
, uint32_t desc
)
757 intptr_t oprsz
= simd_oprsz(desc
);
760 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
761 uint8_t sh
= *(uint64_t *)(b
+ i
) & 63;
762 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) << sh
;
764 clear_high(d
, oprsz
, desc
);
767 void HELPER(gvec_shr8v
)(void *d
, void *a
, void *b
, uint32_t desc
)
769 intptr_t oprsz
= simd_oprsz(desc
);
772 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
773 uint8_t sh
= *(uint8_t *)(b
+ i
) & 7;
774 *(uint8_t *)(d
+ i
) = *(uint8_t *)(a
+ i
) >> sh
;
776 clear_high(d
, oprsz
, desc
);
779 void HELPER(gvec_shr16v
)(void *d
, void *a
, void *b
, uint32_t desc
)
781 intptr_t oprsz
= simd_oprsz(desc
);
784 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
785 uint8_t sh
= *(uint16_t *)(b
+ i
) & 15;
786 *(uint16_t *)(d
+ i
) = *(uint16_t *)(a
+ i
) >> sh
;
788 clear_high(d
, oprsz
, desc
);
791 void HELPER(gvec_shr32v
)(void *d
, void *a
, void *b
, uint32_t desc
)
793 intptr_t oprsz
= simd_oprsz(desc
);
796 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
797 uint8_t sh
= *(uint32_t *)(b
+ i
) & 31;
798 *(uint32_t *)(d
+ i
) = *(uint32_t *)(a
+ i
) >> sh
;
800 clear_high(d
, oprsz
, desc
);
803 void HELPER(gvec_shr64v
)(void *d
, void *a
, void *b
, uint32_t desc
)
805 intptr_t oprsz
= simd_oprsz(desc
);
808 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
809 uint8_t sh
= *(uint64_t *)(b
+ i
) & 63;
810 *(uint64_t *)(d
+ i
) = *(uint64_t *)(a
+ i
) >> sh
;
812 clear_high(d
, oprsz
, desc
);
815 void HELPER(gvec_sar8v
)(void *d
, void *a
, void *b
, uint32_t desc
)
817 intptr_t oprsz
= simd_oprsz(desc
);
820 for (i
= 0; i
< oprsz
; i
+= sizeof(int8_t)) {
821 uint8_t sh
= *(uint8_t *)(b
+ i
) & 7;
822 *(int8_t *)(d
+ i
) = *(int8_t *)(a
+ i
) >> sh
;
824 clear_high(d
, oprsz
, desc
);
827 void HELPER(gvec_sar16v
)(void *d
, void *a
, void *b
, uint32_t desc
)
829 intptr_t oprsz
= simd_oprsz(desc
);
832 for (i
= 0; i
< oprsz
; i
+= sizeof(int16_t)) {
833 uint8_t sh
= *(uint16_t *)(b
+ i
) & 15;
834 *(int16_t *)(d
+ i
) = *(int16_t *)(a
+ i
) >> sh
;
836 clear_high(d
, oprsz
, desc
);
839 void HELPER(gvec_sar32v
)(void *d
, void *a
, void *b
, uint32_t desc
)
841 intptr_t oprsz
= simd_oprsz(desc
);
844 for (i
= 0; i
< oprsz
; i
+= sizeof(int32_t)) {
845 uint8_t sh
= *(uint32_t *)(b
+ i
) & 31;
846 *(int32_t *)(d
+ i
) = *(int32_t *)(a
+ i
) >> sh
;
848 clear_high(d
, oprsz
, desc
);
851 void HELPER(gvec_sar64v
)(void *d
, void *a
, void *b
, uint32_t desc
)
853 intptr_t oprsz
= simd_oprsz(desc
);
856 for (i
= 0; i
< oprsz
; i
+= sizeof(int64_t)) {
857 uint8_t sh
= *(uint64_t *)(b
+ i
) & 63;
858 *(int64_t *)(d
+ i
) = *(int64_t *)(a
+ i
) >> sh
;
860 clear_high(d
, oprsz
, desc
);
863 #define DO_CMP1(NAME, TYPE, OP) \
864 void HELPER(NAME)(void *d, void *a, void *b, uint32_t desc) \
866 intptr_t oprsz = simd_oprsz(desc); \
868 for (i = 0; i < oprsz; i += sizeof(TYPE)) { \
869 *(TYPE *)(d + i) = -(*(TYPE *)(a + i) OP *(TYPE *)(b + i)); \
871 clear_high(d, oprsz, desc); \
874 #define DO_CMP2(SZ) \
875 DO_CMP1(gvec_eq##SZ, uint##SZ##_t, ==) \
876 DO_CMP1(gvec_ne##SZ, uint##SZ##_t, !=) \
877 DO_CMP1(gvec_lt##SZ, int##SZ##_t, <) \
878 DO_CMP1(gvec_le##SZ, int##SZ##_t, <=) \
879 DO_CMP1(gvec_ltu##SZ, uint##SZ##_t, <) \
880 DO_CMP1(gvec_leu##SZ, uint##SZ##_t, <=)
890 void HELPER(gvec_ssadd8
)(void *d
, void *a
, void *b
, uint32_t desc
)
892 intptr_t oprsz
= simd_oprsz(desc
);
895 for (i
= 0; i
< oprsz
; i
+= sizeof(int8_t)) {
896 int r
= *(int8_t *)(a
+ i
) + *(int8_t *)(b
+ i
);
899 } else if (r
< INT8_MIN
) {
902 *(int8_t *)(d
+ i
) = r
;
904 clear_high(d
, oprsz
, desc
);
907 void HELPER(gvec_ssadd16
)(void *d
, void *a
, void *b
, uint32_t desc
)
909 intptr_t oprsz
= simd_oprsz(desc
);
912 for (i
= 0; i
< oprsz
; i
+= sizeof(int16_t)) {
913 int r
= *(int16_t *)(a
+ i
) + *(int16_t *)(b
+ i
);
916 } else if (r
< INT16_MIN
) {
919 *(int16_t *)(d
+ i
) = r
;
921 clear_high(d
, oprsz
, desc
);
924 void HELPER(gvec_ssadd32
)(void *d
, void *a
, void *b
, uint32_t desc
)
926 intptr_t oprsz
= simd_oprsz(desc
);
929 for (i
= 0; i
< oprsz
; i
+= sizeof(int32_t)) {
930 int32_t ai
= *(int32_t *)(a
+ i
);
931 int32_t bi
= *(int32_t *)(b
+ i
);
932 int32_t di
= ai
+ bi
;
933 if (((di
^ ai
) &~ (ai
^ bi
)) < 0) {
934 /* Signed overflow. */
935 di
= (di
< 0 ? INT32_MAX
: INT32_MIN
);
937 *(int32_t *)(d
+ i
) = di
;
939 clear_high(d
, oprsz
, desc
);
942 void HELPER(gvec_ssadd64
)(void *d
, void *a
, void *b
, uint32_t desc
)
944 intptr_t oprsz
= simd_oprsz(desc
);
947 for (i
= 0; i
< oprsz
; i
+= sizeof(int64_t)) {
948 int64_t ai
= *(int64_t *)(a
+ i
);
949 int64_t bi
= *(int64_t *)(b
+ i
);
950 int64_t di
= ai
+ bi
;
951 if (((di
^ ai
) &~ (ai
^ bi
)) < 0) {
952 /* Signed overflow. */
953 di
= (di
< 0 ? INT64_MAX
: INT64_MIN
);
955 *(int64_t *)(d
+ i
) = di
;
957 clear_high(d
, oprsz
, desc
);
960 void HELPER(gvec_sssub8
)(void *d
, void *a
, void *b
, uint32_t desc
)
962 intptr_t oprsz
= simd_oprsz(desc
);
965 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
966 int r
= *(int8_t *)(a
+ i
) - *(int8_t *)(b
+ i
);
969 } else if (r
< INT8_MIN
) {
972 *(uint8_t *)(d
+ i
) = r
;
974 clear_high(d
, oprsz
, desc
);
977 void HELPER(gvec_sssub16
)(void *d
, void *a
, void *b
, uint32_t desc
)
979 intptr_t oprsz
= simd_oprsz(desc
);
982 for (i
= 0; i
< oprsz
; i
+= sizeof(int16_t)) {
983 int r
= *(int16_t *)(a
+ i
) - *(int16_t *)(b
+ i
);
986 } else if (r
< INT16_MIN
) {
989 *(int16_t *)(d
+ i
) = r
;
991 clear_high(d
, oprsz
, desc
);
994 void HELPER(gvec_sssub32
)(void *d
, void *a
, void *b
, uint32_t desc
)
996 intptr_t oprsz
= simd_oprsz(desc
);
999 for (i
= 0; i
< oprsz
; i
+= sizeof(int32_t)) {
1000 int32_t ai
= *(int32_t *)(a
+ i
);
1001 int32_t bi
= *(int32_t *)(b
+ i
);
1002 int32_t di
= ai
- bi
;
1003 if (((di
^ ai
) & (ai
^ bi
)) < 0) {
1004 /* Signed overflow. */
1005 di
= (di
< 0 ? INT32_MAX
: INT32_MIN
);
1007 *(int32_t *)(d
+ i
) = di
;
1009 clear_high(d
, oprsz
, desc
);
1012 void HELPER(gvec_sssub64
)(void *d
, void *a
, void *b
, uint32_t desc
)
1014 intptr_t oprsz
= simd_oprsz(desc
);
1017 for (i
= 0; i
< oprsz
; i
+= sizeof(int64_t)) {
1018 int64_t ai
= *(int64_t *)(a
+ i
);
1019 int64_t bi
= *(int64_t *)(b
+ i
);
1020 int64_t di
= ai
- bi
;
1021 if (((di
^ ai
) & (ai
^ bi
)) < 0) {
1022 /* Signed overflow. */
1023 di
= (di
< 0 ? INT64_MAX
: INT64_MIN
);
1025 *(int64_t *)(d
+ i
) = di
;
1027 clear_high(d
, oprsz
, desc
);
1030 void HELPER(gvec_usadd8
)(void *d
, void *a
, void *b
, uint32_t desc
)
1032 intptr_t oprsz
= simd_oprsz(desc
);
1035 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
1036 unsigned r
= *(uint8_t *)(a
+ i
) + *(uint8_t *)(b
+ i
);
1037 if (r
> UINT8_MAX
) {
1040 *(uint8_t *)(d
+ i
) = r
;
1042 clear_high(d
, oprsz
, desc
);
1045 void HELPER(gvec_usadd16
)(void *d
, void *a
, void *b
, uint32_t desc
)
1047 intptr_t oprsz
= simd_oprsz(desc
);
1050 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
1051 unsigned r
= *(uint16_t *)(a
+ i
) + *(uint16_t *)(b
+ i
);
1052 if (r
> UINT16_MAX
) {
1055 *(uint16_t *)(d
+ i
) = r
;
1057 clear_high(d
, oprsz
, desc
);
1060 void HELPER(gvec_usadd32
)(void *d
, void *a
, void *b
, uint32_t desc
)
1062 intptr_t oprsz
= simd_oprsz(desc
);
1065 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
1066 uint32_t ai
= *(uint32_t *)(a
+ i
);
1067 uint32_t bi
= *(uint32_t *)(b
+ i
);
1068 uint32_t di
= ai
+ bi
;
1072 *(uint32_t *)(d
+ i
) = di
;
1074 clear_high(d
, oprsz
, desc
);
1077 void HELPER(gvec_usadd64
)(void *d
, void *a
, void *b
, uint32_t desc
)
1079 intptr_t oprsz
= simd_oprsz(desc
);
1082 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
1083 uint64_t ai
= *(uint64_t *)(a
+ i
);
1084 uint64_t bi
= *(uint64_t *)(b
+ i
);
1085 uint64_t di
= ai
+ bi
;
1089 *(uint64_t *)(d
+ i
) = di
;
1091 clear_high(d
, oprsz
, desc
);
1094 void HELPER(gvec_ussub8
)(void *d
, void *a
, void *b
, uint32_t desc
)
1096 intptr_t oprsz
= simd_oprsz(desc
);
1099 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
1100 int r
= *(uint8_t *)(a
+ i
) - *(uint8_t *)(b
+ i
);
1104 *(uint8_t *)(d
+ i
) = r
;
1106 clear_high(d
, oprsz
, desc
);
1109 void HELPER(gvec_ussub16
)(void *d
, void *a
, void *b
, uint32_t desc
)
1111 intptr_t oprsz
= simd_oprsz(desc
);
1114 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
1115 int r
= *(uint16_t *)(a
+ i
) - *(uint16_t *)(b
+ i
);
1119 *(uint16_t *)(d
+ i
) = r
;
1121 clear_high(d
, oprsz
, desc
);
1124 void HELPER(gvec_ussub32
)(void *d
, void *a
, void *b
, uint32_t desc
)
1126 intptr_t oprsz
= simd_oprsz(desc
);
1129 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
1130 uint32_t ai
= *(uint32_t *)(a
+ i
);
1131 uint32_t bi
= *(uint32_t *)(b
+ i
);
1132 uint32_t di
= ai
- bi
;
1136 *(uint32_t *)(d
+ i
) = di
;
1138 clear_high(d
, oprsz
, desc
);
1141 void HELPER(gvec_ussub64
)(void *d
, void *a
, void *b
, uint32_t desc
)
1143 intptr_t oprsz
= simd_oprsz(desc
);
1146 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
1147 uint64_t ai
= *(uint64_t *)(a
+ i
);
1148 uint64_t bi
= *(uint64_t *)(b
+ i
);
1149 uint64_t di
= ai
- bi
;
1153 *(uint64_t *)(d
+ i
) = di
;
1155 clear_high(d
, oprsz
, desc
);
1158 void HELPER(gvec_smin8
)(void *d
, void *a
, void *b
, uint32_t desc
)
1160 intptr_t oprsz
= simd_oprsz(desc
);
1163 for (i
= 0; i
< oprsz
; i
+= sizeof(int8_t)) {
1164 int8_t aa
= *(int8_t *)(a
+ i
);
1165 int8_t bb
= *(int8_t *)(b
+ i
);
1166 int8_t dd
= aa
< bb
? aa
: bb
;
1167 *(int8_t *)(d
+ i
) = dd
;
1169 clear_high(d
, oprsz
, desc
);
1172 void HELPER(gvec_smin16
)(void *d
, void *a
, void *b
, uint32_t desc
)
1174 intptr_t oprsz
= simd_oprsz(desc
);
1177 for (i
= 0; i
< oprsz
; i
+= sizeof(int16_t)) {
1178 int16_t aa
= *(int16_t *)(a
+ i
);
1179 int16_t bb
= *(int16_t *)(b
+ i
);
1180 int16_t dd
= aa
< bb
? aa
: bb
;
1181 *(int16_t *)(d
+ i
) = dd
;
1183 clear_high(d
, oprsz
, desc
);
1186 void HELPER(gvec_smin32
)(void *d
, void *a
, void *b
, uint32_t desc
)
1188 intptr_t oprsz
= simd_oprsz(desc
);
1191 for (i
= 0; i
< oprsz
; i
+= sizeof(int32_t)) {
1192 int32_t aa
= *(int32_t *)(a
+ i
);
1193 int32_t bb
= *(int32_t *)(b
+ i
);
1194 int32_t dd
= aa
< bb
? aa
: bb
;
1195 *(int32_t *)(d
+ i
) = dd
;
1197 clear_high(d
, oprsz
, desc
);
1200 void HELPER(gvec_smin64
)(void *d
, void *a
, void *b
, uint32_t desc
)
1202 intptr_t oprsz
= simd_oprsz(desc
);
1205 for (i
= 0; i
< oprsz
; i
+= sizeof(int64_t)) {
1206 int64_t aa
= *(int64_t *)(a
+ i
);
1207 int64_t bb
= *(int64_t *)(b
+ i
);
1208 int64_t dd
= aa
< bb
? aa
: bb
;
1209 *(int64_t *)(d
+ i
) = dd
;
1211 clear_high(d
, oprsz
, desc
);
1214 void HELPER(gvec_smax8
)(void *d
, void *a
, void *b
, uint32_t desc
)
1216 intptr_t oprsz
= simd_oprsz(desc
);
1219 for (i
= 0; i
< oprsz
; i
+= sizeof(int8_t)) {
1220 int8_t aa
= *(int8_t *)(a
+ i
);
1221 int8_t bb
= *(int8_t *)(b
+ i
);
1222 int8_t dd
= aa
> bb
? aa
: bb
;
1223 *(int8_t *)(d
+ i
) = dd
;
1225 clear_high(d
, oprsz
, desc
);
1228 void HELPER(gvec_smax16
)(void *d
, void *a
, void *b
, uint32_t desc
)
1230 intptr_t oprsz
= simd_oprsz(desc
);
1233 for (i
= 0; i
< oprsz
; i
+= sizeof(int16_t)) {
1234 int16_t aa
= *(int16_t *)(a
+ i
);
1235 int16_t bb
= *(int16_t *)(b
+ i
);
1236 int16_t dd
= aa
> bb
? aa
: bb
;
1237 *(int16_t *)(d
+ i
) = dd
;
1239 clear_high(d
, oprsz
, desc
);
1242 void HELPER(gvec_smax32
)(void *d
, void *a
, void *b
, uint32_t desc
)
1244 intptr_t oprsz
= simd_oprsz(desc
);
1247 for (i
= 0; i
< oprsz
; i
+= sizeof(int32_t)) {
1248 int32_t aa
= *(int32_t *)(a
+ i
);
1249 int32_t bb
= *(int32_t *)(b
+ i
);
1250 int32_t dd
= aa
> bb
? aa
: bb
;
1251 *(int32_t *)(d
+ i
) = dd
;
1253 clear_high(d
, oprsz
, desc
);
1256 void HELPER(gvec_smax64
)(void *d
, void *a
, void *b
, uint32_t desc
)
1258 intptr_t oprsz
= simd_oprsz(desc
);
1261 for (i
= 0; i
< oprsz
; i
+= sizeof(int64_t)) {
1262 int64_t aa
= *(int64_t *)(a
+ i
);
1263 int64_t bb
= *(int64_t *)(b
+ i
);
1264 int64_t dd
= aa
> bb
? aa
: bb
;
1265 *(int64_t *)(d
+ i
) = dd
;
1267 clear_high(d
, oprsz
, desc
);
1270 void HELPER(gvec_umin8
)(void *d
, void *a
, void *b
, uint32_t desc
)
1272 intptr_t oprsz
= simd_oprsz(desc
);
1275 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
1276 uint8_t aa
= *(uint8_t *)(a
+ i
);
1277 uint8_t bb
= *(uint8_t *)(b
+ i
);
1278 uint8_t dd
= aa
< bb
? aa
: bb
;
1279 *(uint8_t *)(d
+ i
) = dd
;
1281 clear_high(d
, oprsz
, desc
);
1284 void HELPER(gvec_umin16
)(void *d
, void *a
, void *b
, uint32_t desc
)
1286 intptr_t oprsz
= simd_oprsz(desc
);
1289 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
1290 uint16_t aa
= *(uint16_t *)(a
+ i
);
1291 uint16_t bb
= *(uint16_t *)(b
+ i
);
1292 uint16_t dd
= aa
< bb
? aa
: bb
;
1293 *(uint16_t *)(d
+ i
) = dd
;
1295 clear_high(d
, oprsz
, desc
);
1298 void HELPER(gvec_umin32
)(void *d
, void *a
, void *b
, uint32_t desc
)
1300 intptr_t oprsz
= simd_oprsz(desc
);
1303 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
1304 uint32_t aa
= *(uint32_t *)(a
+ i
);
1305 uint32_t bb
= *(uint32_t *)(b
+ i
);
1306 uint32_t dd
= aa
< bb
? aa
: bb
;
1307 *(uint32_t *)(d
+ i
) = dd
;
1309 clear_high(d
, oprsz
, desc
);
1312 void HELPER(gvec_umin64
)(void *d
, void *a
, void *b
, uint32_t desc
)
1314 intptr_t oprsz
= simd_oprsz(desc
);
1317 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
1318 uint64_t aa
= *(uint64_t *)(a
+ i
);
1319 uint64_t bb
= *(uint64_t *)(b
+ i
);
1320 uint64_t dd
= aa
< bb
? aa
: bb
;
1321 *(uint64_t *)(d
+ i
) = dd
;
1323 clear_high(d
, oprsz
, desc
);
1326 void HELPER(gvec_umax8
)(void *d
, void *a
, void *b
, uint32_t desc
)
1328 intptr_t oprsz
= simd_oprsz(desc
);
1331 for (i
= 0; i
< oprsz
; i
+= sizeof(uint8_t)) {
1332 uint8_t aa
= *(uint8_t *)(a
+ i
);
1333 uint8_t bb
= *(uint8_t *)(b
+ i
);
1334 uint8_t dd
= aa
> bb
? aa
: bb
;
1335 *(uint8_t *)(d
+ i
) = dd
;
1337 clear_high(d
, oprsz
, desc
);
1340 void HELPER(gvec_umax16
)(void *d
, void *a
, void *b
, uint32_t desc
)
1342 intptr_t oprsz
= simd_oprsz(desc
);
1345 for (i
= 0; i
< oprsz
; i
+= sizeof(uint16_t)) {
1346 uint16_t aa
= *(uint16_t *)(a
+ i
);
1347 uint16_t bb
= *(uint16_t *)(b
+ i
);
1348 uint16_t dd
= aa
> bb
? aa
: bb
;
1349 *(uint16_t *)(d
+ i
) = dd
;
1351 clear_high(d
, oprsz
, desc
);
1354 void HELPER(gvec_umax32
)(void *d
, void *a
, void *b
, uint32_t desc
)
1356 intptr_t oprsz
= simd_oprsz(desc
);
1359 for (i
= 0; i
< oprsz
; i
+= sizeof(uint32_t)) {
1360 uint32_t aa
= *(uint32_t *)(a
+ i
);
1361 uint32_t bb
= *(uint32_t *)(b
+ i
);
1362 uint32_t dd
= aa
> bb
? aa
: bb
;
1363 *(uint32_t *)(d
+ i
) = dd
;
1365 clear_high(d
, oprsz
, desc
);
1368 void HELPER(gvec_umax64
)(void *d
, void *a
, void *b
, uint32_t desc
)
1370 intptr_t oprsz
= simd_oprsz(desc
);
1373 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
1374 uint64_t aa
= *(uint64_t *)(a
+ i
);
1375 uint64_t bb
= *(uint64_t *)(b
+ i
);
1376 uint64_t dd
= aa
> bb
? aa
: bb
;
1377 *(uint64_t *)(d
+ i
) = dd
;
1379 clear_high(d
, oprsz
, desc
);
1382 void HELPER(gvec_bitsel
)(void *d
, void *a
, void *b
, void *c
, uint32_t desc
)
1384 intptr_t oprsz
= simd_oprsz(desc
);
1387 for (i
= 0; i
< oprsz
; i
+= sizeof(uint64_t)) {
1388 uint64_t aa
= *(uint64_t *)(a
+ i
);
1389 uint64_t bb
= *(uint64_t *)(b
+ i
);
1390 uint64_t cc
= *(uint64_t *)(c
+ i
);
1391 *(uint64_t *)(d
+ i
) = (bb
& aa
) | (cc
& ~aa
);
1393 clear_high(d
, oprsz
, desc
);