?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
atomic_template: add inline trace/plugin helpers
[qemu/armbru.git] / tcg / tcg-op-gvec.c
blob5c95ecd51cad57b67addd814b223f82c59f63159
1 /*
2 * Generic vector operation expansion
3 *
4 * Copyright (c) 2018 Linaro
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "tcg.h"
22 #include "tcg-op.h"
23 #include "tcg-op-gvec.h"
24 #include "qemu/main-loop.h"
25 #include "tcg-gvec-desc.h"
26
27 #define MAX_UNROLL 4
28
29 #ifdef CONFIG_DEBUG_TCG
30 static const TCGOpcode vecop_list_empty[1] = { 0 };
31 #else
32 #define vecop_list_empty NULL
33 #endif
34
35
36 /* Verify vector size and alignment rules. OFS should be the OR of all
37 of the operand offsets so that we can check them all at once. */
38 static void check_size_align(uint32_t oprsz, uint32_t maxsz, uint32_t ofs)
39 {
40 uint32_t opr_align = oprsz >= 16 ? 15 : 7;
41 uint32_t max_align = maxsz >= 16 || oprsz >= 16 ? 15 : 7;
42 tcg_debug_assert(oprsz > 0);
43 tcg_debug_assert(oprsz <= maxsz);
44 tcg_debug_assert((oprsz & opr_align) == 0);
45 tcg_debug_assert((maxsz & max_align) == 0);
46 tcg_debug_assert((ofs & max_align) == 0);
47 }
48
49 /* Verify vector overlap rules for two operands. */
50 static void check_overlap_2(uint32_t d, uint32_t a, uint32_t s)
51 {
52 tcg_debug_assert(d == a || d + s <= a || a + s <= d);
53 }
54
55 /* Verify vector overlap rules for three operands. */
56 static void check_overlap_3(uint32_t d, uint32_t a, uint32_t b, uint32_t s)
57 {
58 check_overlap_2(d, a, s);
59 check_overlap_2(d, b, s);
60 check_overlap_2(a, b, s);
61 }
62
63 /* Verify vector overlap rules for four operands. */
64 static void check_overlap_4(uint32_t d, uint32_t a, uint32_t b,
65 uint32_t c, uint32_t s)
66 {
67 check_overlap_2(d, a, s);
68 check_overlap_2(d, b, s);
69 check_overlap_2(d, c, s);
70 check_overlap_2(a, b, s);
71 check_overlap_2(a, c, s);
72 check_overlap_2(b, c, s);
73 }
74
75 /* Create a descriptor from components. */
76 uint32_t simd_desc(uint32_t oprsz, uint32_t maxsz, int32_t data)
77 {
78 uint32_t desc = 0;
79
80 assert(oprsz % 8 == 0 && oprsz <= (8 << SIMD_OPRSZ_BITS));
81 assert(maxsz % 8 == 0 && maxsz <= (8 << SIMD_MAXSZ_BITS));
82 assert(data == sextract32(data, 0, SIMD_DATA_BITS));
83
84 oprsz = (oprsz / 8) - 1;
85 maxsz = (maxsz / 8) - 1;
86 desc = deposit32(desc, SIMD_OPRSZ_SHIFT, SIMD_OPRSZ_BITS, oprsz);
87 desc = deposit32(desc, SIMD_MAXSZ_SHIFT, SIMD_MAXSZ_BITS, maxsz);
88 desc = deposit32(desc, SIMD_DATA_SHIFT, SIMD_DATA_BITS, data);
89
90 return desc;
91 }
92
93 /* Generate a call to a gvec-style helper with two vector operands. */
94 void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs,
95 uint32_t oprsz, uint32_t maxsz, int32_t data,
96 gen_helper_gvec_2 *fn)
97 {
98 TCGv_ptr a0, a1;
99 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
100
101 a0 = tcg_temp_new_ptr();
102 a1 = tcg_temp_new_ptr();
103
104 tcg_gen_addi_ptr(a0, cpu_env, dofs);
105 tcg_gen_addi_ptr(a1, cpu_env, aofs);
106
107 fn(a0, a1, desc);
108
109 tcg_temp_free_ptr(a0);
110 tcg_temp_free_ptr(a1);
111 tcg_temp_free_i32(desc);
112 }
113
114 /* Generate a call to a gvec-style helper with two vector operands
115 and one scalar operand. */
116 void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c,
117 uint32_t oprsz, uint32_t maxsz, int32_t data,
118 gen_helper_gvec_2i *fn)
119 {
120 TCGv_ptr a0, a1;
121 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
122
123 a0 = tcg_temp_new_ptr();
124 a1 = tcg_temp_new_ptr();
125
126 tcg_gen_addi_ptr(a0, cpu_env, dofs);
127 tcg_gen_addi_ptr(a1, cpu_env, aofs);
128
129 fn(a0, a1, c, desc);
130
131 tcg_temp_free_ptr(a0);
132 tcg_temp_free_ptr(a1);
133 tcg_temp_free_i32(desc);
134 }
135
136 /* Generate a call to a gvec-style helper with three vector operands. */
137 void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
138 uint32_t oprsz, uint32_t maxsz, int32_t data,
139 gen_helper_gvec_3 *fn)
140 {
141 TCGv_ptr a0, a1, a2;
142 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
143
144 a0 = tcg_temp_new_ptr();
145 a1 = tcg_temp_new_ptr();
146 a2 = tcg_temp_new_ptr();
147
148 tcg_gen_addi_ptr(a0, cpu_env, dofs);
149 tcg_gen_addi_ptr(a1, cpu_env, aofs);
150 tcg_gen_addi_ptr(a2, cpu_env, bofs);
151
152 fn(a0, a1, a2, desc);
153
154 tcg_temp_free_ptr(a0);
155 tcg_temp_free_ptr(a1);
156 tcg_temp_free_ptr(a2);
157 tcg_temp_free_i32(desc);
158 }
159
160 /* Generate a call to a gvec-style helper with four vector operands. */
161 void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
162 uint32_t cofs, uint32_t oprsz, uint32_t maxsz,
163 int32_t data, gen_helper_gvec_4 *fn)
164 {
165 TCGv_ptr a0, a1, a2, a3;
166 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
167
168 a0 = tcg_temp_new_ptr();
169 a1 = tcg_temp_new_ptr();
170 a2 = tcg_temp_new_ptr();
171 a3 = tcg_temp_new_ptr();
172
173 tcg_gen_addi_ptr(a0, cpu_env, dofs);
174 tcg_gen_addi_ptr(a1, cpu_env, aofs);
175 tcg_gen_addi_ptr(a2, cpu_env, bofs);
176 tcg_gen_addi_ptr(a3, cpu_env, cofs);
177
178 fn(a0, a1, a2, a3, desc);
179
180 tcg_temp_free_ptr(a0);
181 tcg_temp_free_ptr(a1);
182 tcg_temp_free_ptr(a2);
183 tcg_temp_free_ptr(a3);
184 tcg_temp_free_i32(desc);
185 }
186
187 /* Generate a call to a gvec-style helper with five vector operands. */
188 void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
189 uint32_t cofs, uint32_t xofs, uint32_t oprsz,
190 uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn)
191 {
192 TCGv_ptr a0, a1, a2, a3, a4;
193 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
194
195 a0 = tcg_temp_new_ptr();
196 a1 = tcg_temp_new_ptr();
197 a2 = tcg_temp_new_ptr();
198 a3 = tcg_temp_new_ptr();
199 a4 = tcg_temp_new_ptr();
200
201 tcg_gen_addi_ptr(a0, cpu_env, dofs);
202 tcg_gen_addi_ptr(a1, cpu_env, aofs);
203 tcg_gen_addi_ptr(a2, cpu_env, bofs);
204 tcg_gen_addi_ptr(a3, cpu_env, cofs);
205 tcg_gen_addi_ptr(a4, cpu_env, xofs);
206
207 fn(a0, a1, a2, a3, a4, desc);
208
209 tcg_temp_free_ptr(a0);
210 tcg_temp_free_ptr(a1);
211 tcg_temp_free_ptr(a2);
212 tcg_temp_free_ptr(a3);
213 tcg_temp_free_ptr(a4);
214 tcg_temp_free_i32(desc);
215 }
216
217 /* Generate a call to a gvec-style helper with three vector operands
218 and an extra pointer operand. */
219 void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs,
220 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
221 int32_t data, gen_helper_gvec_2_ptr *fn)
222 {
223 TCGv_ptr a0, a1;
224 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
225
226 a0 = tcg_temp_new_ptr();
227 a1 = tcg_temp_new_ptr();
228
229 tcg_gen_addi_ptr(a0, cpu_env, dofs);
230 tcg_gen_addi_ptr(a1, cpu_env, aofs);
231
232 fn(a0, a1, ptr, desc);
233
234 tcg_temp_free_ptr(a0);
235 tcg_temp_free_ptr(a1);
236 tcg_temp_free_i32(desc);
237 }
238
239 /* Generate a call to a gvec-style helper with three vector operands
240 and an extra pointer operand. */
241 void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
242 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
243 int32_t data, gen_helper_gvec_3_ptr *fn)
244 {
245 TCGv_ptr a0, a1, a2;
246 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
247
248 a0 = tcg_temp_new_ptr();
249 a1 = tcg_temp_new_ptr();
250 a2 = tcg_temp_new_ptr();
251
252 tcg_gen_addi_ptr(a0, cpu_env, dofs);
253 tcg_gen_addi_ptr(a1, cpu_env, aofs);
254 tcg_gen_addi_ptr(a2, cpu_env, bofs);
255
256 fn(a0, a1, a2, ptr, desc);
257
258 tcg_temp_free_ptr(a0);
259 tcg_temp_free_ptr(a1);
260 tcg_temp_free_ptr(a2);
261 tcg_temp_free_i32(desc);
262 }
263
264 /* Generate a call to a gvec-style helper with four vector operands
265 and an extra pointer operand. */
266 void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
267 uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz,
268 uint32_t maxsz, int32_t data,
269 gen_helper_gvec_4_ptr *fn)
270 {
271 TCGv_ptr a0, a1, a2, a3;
272 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
273
274 a0 = tcg_temp_new_ptr();
275 a1 = tcg_temp_new_ptr();
276 a2 = tcg_temp_new_ptr();
277 a3 = tcg_temp_new_ptr();
278
279 tcg_gen_addi_ptr(a0, cpu_env, dofs);
280 tcg_gen_addi_ptr(a1, cpu_env, aofs);
281 tcg_gen_addi_ptr(a2, cpu_env, bofs);
282 tcg_gen_addi_ptr(a3, cpu_env, cofs);
283
284 fn(a0, a1, a2, a3, ptr, desc);
285
286 tcg_temp_free_ptr(a0);
287 tcg_temp_free_ptr(a1);
288 tcg_temp_free_ptr(a2);
289 tcg_temp_free_ptr(a3);
290 tcg_temp_free_i32(desc);
291 }
292
293 /* Return true if we want to implement something of OPRSZ bytes
294 in units of LNSZ. This limits the expansion of inline code. */
295 static inline bool check_size_impl(uint32_t oprsz, uint32_t lnsz)
296 {
297 if (oprsz % lnsz == 0) {
298 uint32_t lnct = oprsz / lnsz;
299 return lnct >= 1 && lnct <= MAX_UNROLL;
300 }
301 return false;
302 }
303
304 static void expand_clr(uint32_t dofs, uint32_t maxsz);
305
306 /* Duplicate C as per VECE. */
307 uint64_t (dup_const)(unsigned vece, uint64_t c)
308 {
309 switch (vece) {
310 case MO_8:
311 return 0x0101010101010101ull * (uint8_t)c;
312 case MO_16:
313 return 0x0001000100010001ull * (uint16_t)c;
314 case MO_32:
315 return 0x0000000100000001ull * (uint32_t)c;
316 case MO_64:
317 return c;
318 default:
319 g_assert_not_reached();
320 }
321 }
322
323 /* Duplicate IN into OUT as per VECE. */
324 static void gen_dup_i32(unsigned vece, TCGv_i32 out, TCGv_i32 in)
325 {
326 switch (vece) {
327 case MO_8:
328 tcg_gen_ext8u_i32(out, in);
329 tcg_gen_muli_i32(out, out, 0x01010101);
330 break;
331 case MO_16:
332 tcg_gen_deposit_i32(out, in, in, 16, 16);
333 break;
334 case MO_32:
335 tcg_gen_mov_i32(out, in);
336 break;
337 default:
338 g_assert_not_reached();
339 }
340 }
341
342 static void gen_dup_i64(unsigned vece, TCGv_i64 out, TCGv_i64 in)
343 {
344 switch (vece) {
345 case MO_8:
346 tcg_gen_ext8u_i64(out, in);
347 tcg_gen_muli_i64(out, out, 0x0101010101010101ull);
348 break;
349 case MO_16:
350 tcg_gen_ext16u_i64(out, in);
351 tcg_gen_muli_i64(out, out, 0x0001000100010001ull);
352 break;
353 case MO_32:
354 tcg_gen_deposit_i64(out, in, in, 32, 32);
355 break;
356 case MO_64:
357 tcg_gen_mov_i64(out, in);
358 break;
359 default:
360 g_assert_not_reached();
361 }
362 }
363
364 /* Select a supported vector type for implementing an operation on SIZE
365 * bytes. If OP is 0, assume that the real operation to be performed is
366 * required by all backends. Otherwise, make sure than OP can be performed
367 * on elements of size VECE in the selected type. Do not select V64 if
368 * PREFER_I64 is true. Return 0 if no vector type is selected.
369 */
370 static TCGType choose_vector_type(const TCGOpcode *list, unsigned vece,
371 uint32_t size, bool prefer_i64)
372 {
373 if (TCG_TARGET_HAS_v256 && check_size_impl(size, 32)) {
374 /*
375 * Recall that ARM SVE allows vector sizes that are not a
376 * power of 2, but always a multiple of 16. The intent is
377 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
378 * It is hard to imagine a case in which v256 is supported
379 * but v128 is not, but check anyway.
380 */
381 if (tcg_can_emit_vecop_list(list, TCG_TYPE_V256, vece)
382 && (size % 32 == 0
383 || tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece))) {
384 return TCG_TYPE_V256;
385 }
386 }
387 if (TCG_TARGET_HAS_v128 && check_size_impl(size, 16)
388 && tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece)) {
389 return TCG_TYPE_V128;
390 }
391 if (TCG_TARGET_HAS_v64 && !prefer_i64 && check_size_impl(size, 8)
392 && tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)) {
393 return TCG_TYPE_V64;
394 }
395 return 0;
396 }
397
398 static void do_dup_store(TCGType type, uint32_t dofs, uint32_t oprsz,
399 uint32_t maxsz, TCGv_vec t_vec)
400 {
401 uint32_t i = 0;
402
403 switch (type) {
404 case TCG_TYPE_V256:
405 /*
406 * Recall that ARM SVE allows vector sizes that are not a
407 * power of 2, but always a multiple of 16. The intent is
408 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
409 */
410 for (; i + 32 <= oprsz; i += 32) {
411 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V256);
412 }
413 /* fallthru */
414 case TCG_TYPE_V128:
415 for (; i + 16 <= oprsz; i += 16) {
416 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V128);
417 }
418 break;
419 case TCG_TYPE_V64:
420 for (; i < oprsz; i += 8) {
421 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V64);
422 }
423 break;
424 default:
425 g_assert_not_reached();
426 }
427
428 if (oprsz < maxsz) {
429 expand_clr(dofs + oprsz, maxsz - oprsz);
430 }
431 }
432
433 /* Set OPRSZ bytes at DOFS to replications of IN_32, IN_64 or IN_C.
434 * Only one of IN_32 or IN_64 may be set;
435 * IN_C is used if IN_32 and IN_64 are unset.
436 */
437 static void do_dup(unsigned vece, uint32_t dofs, uint32_t oprsz,
438 uint32_t maxsz, TCGv_i32 in_32, TCGv_i64 in_64,
439 uint64_t in_c)
440 {
441 TCGType type;
442 TCGv_i64 t_64;
443 TCGv_i32 t_32, t_desc;
444 TCGv_ptr t_ptr;
445 uint32_t i;
446
447 assert(vece <= (in_32 ? MO_32 : MO_64));
448 assert(in_32 == NULL || in_64 == NULL);
449
450 /* If we're storing 0, expand oprsz to maxsz. */
451 if (in_32 == NULL && in_64 == NULL) {
452 in_c = dup_const(vece, in_c);
453 if (in_c == 0) {
454 oprsz = maxsz;
455 }
456 }
457
458 /* Implement inline with a vector type, if possible.
459 * Prefer integer when 64-bit host and no variable dup.
460 */
461 type = choose_vector_type(NULL, vece, oprsz,
462 (TCG_TARGET_REG_BITS == 64 && in_32 == NULL
463 && (in_64 == NULL || vece == MO_64)));
464 if (type != 0) {
465 TCGv_vec t_vec = tcg_temp_new_vec(type);
466
467 if (in_32) {
468 tcg_gen_dup_i32_vec(vece, t_vec, in_32);
469 } else if (in_64) {
470 tcg_gen_dup_i64_vec(vece, t_vec, in_64);
471 } else {
472 tcg_gen_dupi_vec(vece, t_vec, in_c);
473 }
474 do_dup_store(type, dofs, oprsz, maxsz, t_vec);
475 tcg_temp_free_vec(t_vec);
476 return;
477 }
478
479 /* Otherwise, inline with an integer type, unless "large". */
480 if (check_size_impl(oprsz, TCG_TARGET_REG_BITS / 8)) {
481 t_64 = NULL;
482 t_32 = NULL;
483
484 if (in_32) {
485 /* We are given a 32-bit variable input. For a 64-bit host,
486 use a 64-bit operation unless the 32-bit operation would
487 be simple enough. */
488 if (TCG_TARGET_REG_BITS == 64
489 && (vece != MO_32 || !check_size_impl(oprsz, 4))) {
490 t_64 = tcg_temp_new_i64();
491 tcg_gen_extu_i32_i64(t_64, in_32);
492 gen_dup_i64(vece, t_64, t_64);
493 } else {
494 t_32 = tcg_temp_new_i32();
495 gen_dup_i32(vece, t_32, in_32);
496 }
497 } else if (in_64) {
498 /* We are given a 64-bit variable input. */
499 t_64 = tcg_temp_new_i64();
500 gen_dup_i64(vece, t_64, in_64);
501 } else {
502 /* We are given a constant input. */
503 /* For 64-bit hosts, use 64-bit constants for "simple" constants
504 or when we'd need too many 32-bit stores, or when a 64-bit
505 constant is really required. */
506 if (vece == MO_64
507 || (TCG_TARGET_REG_BITS == 64
508 && (in_c == 0 || in_c == -1
509 || !check_size_impl(oprsz, 4)))) {
510 t_64 = tcg_const_i64(in_c);
511 } else {
512 t_32 = tcg_const_i32(in_c);
513 }
514 }
515
516 /* Implement inline if we picked an implementation size above. */
517 if (t_32) {
518 for (i = 0; i < oprsz; i += 4) {
519 tcg_gen_st_i32(t_32, cpu_env, dofs + i);
520 }
521 tcg_temp_free_i32(t_32);
522 goto done;
523 }
524 if (t_64) {
525 for (i = 0; i < oprsz; i += 8) {
526 tcg_gen_st_i64(t_64, cpu_env, dofs + i);
527 }
528 tcg_temp_free_i64(t_64);
529 goto done;
530 }
531 }
532
533 /* Otherwise implement out of line. */
534 t_ptr = tcg_temp_new_ptr();
535 tcg_gen_addi_ptr(t_ptr, cpu_env, dofs);
536 t_desc = tcg_const_i32(simd_desc(oprsz, maxsz, 0));
537
538 if (vece == MO_64) {
539 if (in_64) {
540 gen_helper_gvec_dup64(t_ptr, t_desc, in_64);
541 } else {
542 t_64 = tcg_const_i64(in_c);
543 gen_helper_gvec_dup64(t_ptr, t_desc, t_64);
544 tcg_temp_free_i64(t_64);
545 }
546 } else {
547 typedef void dup_fn(TCGv_ptr, TCGv_i32, TCGv_i32);
548 static dup_fn * const fns[3] = {
549 gen_helper_gvec_dup8,
550 gen_helper_gvec_dup16,
551 gen_helper_gvec_dup32
552 };
553
554 if (in_32) {
555 fns[vece](t_ptr, t_desc, in_32);
556 } else {
557 t_32 = tcg_temp_new_i32();
558 if (in_64) {
559 tcg_gen_extrl_i64_i32(t_32, in_64);
560 } else if (vece == MO_8) {
561 tcg_gen_movi_i32(t_32, in_c & 0xff);
562 } else if (vece == MO_16) {
563 tcg_gen_movi_i32(t_32, in_c & 0xffff);
564 } else {
565 tcg_gen_movi_i32(t_32, in_c);
566 }
567 fns[vece](t_ptr, t_desc, t_32);
568 tcg_temp_free_i32(t_32);
569 }
570 }
571
572 tcg_temp_free_ptr(t_ptr);
573 tcg_temp_free_i32(t_desc);
574 return;
575
576 done:
577 if (oprsz < maxsz) {
578 expand_clr(dofs + oprsz, maxsz - oprsz);
579 }
580 }
581
582 /* Likewise, but with zero. */
583 static void expand_clr(uint32_t dofs, uint32_t maxsz)
584 {
585 do_dup(MO_8, dofs, maxsz, maxsz, NULL, NULL, 0);
586 }
587
588 /* Expand OPSZ bytes worth of two-operand operations using i32 elements. */
589 static void expand_2_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
590 void (*fni)(TCGv_i32, TCGv_i32))
591 {
592 TCGv_i32 t0 = tcg_temp_new_i32();
593 uint32_t i;
594
595 for (i = 0; i < oprsz; i += 4) {
596 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
597 fni(t0, t0);
598 tcg_gen_st_i32(t0, cpu_env, dofs + i);
599 }
600 tcg_temp_free_i32(t0);
601 }
602
603 static void expand_2i_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
604 int32_t c, bool load_dest,
605 void (*fni)(TCGv_i32, TCGv_i32, int32_t))
606 {
607 TCGv_i32 t0 = tcg_temp_new_i32();
608 TCGv_i32 t1 = tcg_temp_new_i32();
609 uint32_t i;
610
611 for (i = 0; i < oprsz; i += 4) {
612 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
613 if (load_dest) {
614 tcg_gen_ld_i32(t1, cpu_env, dofs + i);
615 }
616 fni(t1, t0, c);
617 tcg_gen_st_i32(t1, cpu_env, dofs + i);
618 }
619 tcg_temp_free_i32(t0);
620 tcg_temp_free_i32(t1);
621 }
622
623 static void expand_2s_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
624 TCGv_i32 c, bool scalar_first,
625 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
626 {
627 TCGv_i32 t0 = tcg_temp_new_i32();
628 TCGv_i32 t1 = tcg_temp_new_i32();
629 uint32_t i;
630
631 for (i = 0; i < oprsz; i += 4) {
632 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
633 if (scalar_first) {
634 fni(t1, c, t0);
635 } else {
636 fni(t1, t0, c);
637 }
638 tcg_gen_st_i32(t1, cpu_env, dofs + i);
639 }
640 tcg_temp_free_i32(t0);
641 tcg_temp_free_i32(t1);
642 }
643
644 /* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
645 static void expand_3_i32(uint32_t dofs, uint32_t aofs,
646 uint32_t bofs, uint32_t oprsz, bool load_dest,
647 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
648 {
649 TCGv_i32 t0 = tcg_temp_new_i32();
650 TCGv_i32 t1 = tcg_temp_new_i32();
651 TCGv_i32 t2 = tcg_temp_new_i32();
652 uint32_t i;
653
654 for (i = 0; i < oprsz; i += 4) {
655 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
656 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
657 if (load_dest) {
658 tcg_gen_ld_i32(t2, cpu_env, dofs + i);
659 }
660 fni(t2, t0, t1);
661 tcg_gen_st_i32(t2, cpu_env, dofs + i);
662 }
663 tcg_temp_free_i32(t2);
664 tcg_temp_free_i32(t1);
665 tcg_temp_free_i32(t0);
666 }
667
668 static void expand_3i_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
669 uint32_t oprsz, int32_t c, bool load_dest,
670 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t))
671 {
672 TCGv_i32 t0 = tcg_temp_new_i32();
673 TCGv_i32 t1 = tcg_temp_new_i32();
674 TCGv_i32 t2 = tcg_temp_new_i32();
675 uint32_t i;
676
677 for (i = 0; i < oprsz; i += 4) {
678 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
679 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
680 if (load_dest) {
681 tcg_gen_ld_i32(t2, cpu_env, dofs + i);
682 }
683 fni(t2, t0, t1, c);
684 tcg_gen_st_i32(t2, cpu_env, dofs + i);
685 }
686 tcg_temp_free_i32(t0);
687 tcg_temp_free_i32(t1);
688 tcg_temp_free_i32(t2);
689 }
690
691 /* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
692 static void expand_4_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
693 uint32_t cofs, uint32_t oprsz, bool write_aofs,
694 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32))
695 {
696 TCGv_i32 t0 = tcg_temp_new_i32();
697 TCGv_i32 t1 = tcg_temp_new_i32();
698 TCGv_i32 t2 = tcg_temp_new_i32();
699 TCGv_i32 t3 = tcg_temp_new_i32();
700 uint32_t i;
701
702 for (i = 0; i < oprsz; i += 4) {
703 tcg_gen_ld_i32(t1, cpu_env, aofs + i);
704 tcg_gen_ld_i32(t2, cpu_env, bofs + i);
705 tcg_gen_ld_i32(t3, cpu_env, cofs + i);
706 fni(t0, t1, t2, t3);
707 tcg_gen_st_i32(t0, cpu_env, dofs + i);
708 if (write_aofs) {
709 tcg_gen_st_i32(t1, cpu_env, aofs + i);
710 }
711 }
712 tcg_temp_free_i32(t3);
713 tcg_temp_free_i32(t2);
714 tcg_temp_free_i32(t1);
715 tcg_temp_free_i32(t0);
716 }
717
718 /* Expand OPSZ bytes worth of two-operand operations using i64 elements. */
719 static void expand_2_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
720 void (*fni)(TCGv_i64, TCGv_i64))
721 {
722 TCGv_i64 t0 = tcg_temp_new_i64();
723 uint32_t i;
724
725 for (i = 0; i < oprsz; i += 8) {
726 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
727 fni(t0, t0);
728 tcg_gen_st_i64(t0, cpu_env, dofs + i);
729 }
730 tcg_temp_free_i64(t0);
731 }
732
733 static void expand_2i_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
734 int64_t c, bool load_dest,
735 void (*fni)(TCGv_i64, TCGv_i64, int64_t))
736 {
737 TCGv_i64 t0 = tcg_temp_new_i64();
738 TCGv_i64 t1 = tcg_temp_new_i64();
739 uint32_t i;
740
741 for (i = 0; i < oprsz; i += 8) {
742 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
743 if (load_dest) {
744 tcg_gen_ld_i64(t1, cpu_env, dofs + i);
745 }
746 fni(t1, t0, c);
747 tcg_gen_st_i64(t1, cpu_env, dofs + i);
748 }
749 tcg_temp_free_i64(t0);
750 tcg_temp_free_i64(t1);
751 }
752
753 static void expand_2s_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
754 TCGv_i64 c, bool scalar_first,
755 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
756 {
757 TCGv_i64 t0 = tcg_temp_new_i64();
758 TCGv_i64 t1 = tcg_temp_new_i64();
759 uint32_t i;
760
761 for (i = 0; i < oprsz; i += 8) {
762 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
763 if (scalar_first) {
764 fni(t1, c, t0);
765 } else {
766 fni(t1, t0, c);
767 }
768 tcg_gen_st_i64(t1, cpu_env, dofs + i);
769 }
770 tcg_temp_free_i64(t0);
771 tcg_temp_free_i64(t1);
772 }
773
774 /* Expand OPSZ bytes worth of three-operand operations using i64 elements. */
775 static void expand_3_i64(uint32_t dofs, uint32_t aofs,
776 uint32_t bofs, uint32_t oprsz, bool load_dest,
777 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
778 {
779 TCGv_i64 t0 = tcg_temp_new_i64();
780 TCGv_i64 t1 = tcg_temp_new_i64();
781 TCGv_i64 t2 = tcg_temp_new_i64();
782 uint32_t i;
783
784 for (i = 0; i < oprsz; i += 8) {
785 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
786 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
787 if (load_dest) {
788 tcg_gen_ld_i64(t2, cpu_env, dofs + i);
789 }
790 fni(t2, t0, t1);
791 tcg_gen_st_i64(t2, cpu_env, dofs + i);
792 }
793 tcg_temp_free_i64(t2);
794 tcg_temp_free_i64(t1);
795 tcg_temp_free_i64(t0);
796 }
797
798 static void expand_3i_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
799 uint32_t oprsz, int64_t c, bool load_dest,
800 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t))
801 {
802 TCGv_i64 t0 = tcg_temp_new_i64();
803 TCGv_i64 t1 = tcg_temp_new_i64();
804 TCGv_i64 t2 = tcg_temp_new_i64();
805 uint32_t i;
806
807 for (i = 0; i < oprsz; i += 8) {
808 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
809 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
810 if (load_dest) {
811 tcg_gen_ld_i64(t2, cpu_env, dofs + i);
812 }
813 fni(t2, t0, t1, c);
814 tcg_gen_st_i64(t2, cpu_env, dofs + i);
815 }
816 tcg_temp_free_i64(t0);
817 tcg_temp_free_i64(t1);
818 tcg_temp_free_i64(t2);
819 }
820
821 /* Expand OPSZ bytes worth of three-operand operations using i64 elements. */
822 static void expand_4_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
823 uint32_t cofs, uint32_t oprsz, bool write_aofs,
824 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64))
825 {
826 TCGv_i64 t0 = tcg_temp_new_i64();
827 TCGv_i64 t1 = tcg_temp_new_i64();
828 TCGv_i64 t2 = tcg_temp_new_i64();
829 TCGv_i64 t3 = tcg_temp_new_i64();
830 uint32_t i;
831
832 for (i = 0; i < oprsz; i += 8) {
833 tcg_gen_ld_i64(t1, cpu_env, aofs + i);
834 tcg_gen_ld_i64(t2, cpu_env, bofs + i);
835 tcg_gen_ld_i64(t3, cpu_env, cofs + i);
836 fni(t0, t1, t2, t3);
837 tcg_gen_st_i64(t0, cpu_env, dofs + i);
838 if (write_aofs) {
839 tcg_gen_st_i64(t1, cpu_env, aofs + i);
840 }
841 }
842 tcg_temp_free_i64(t3);
843 tcg_temp_free_i64(t2);
844 tcg_temp_free_i64(t1);
845 tcg_temp_free_i64(t0);
846 }
847
848 /* Expand OPSZ bytes worth of two-operand operations using host vectors. */
849 static void expand_2_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
850 uint32_t oprsz, uint32_t tysz, TCGType type,
851 void (*fni)(unsigned, TCGv_vec, TCGv_vec))
852 {
853 TCGv_vec t0 = tcg_temp_new_vec(type);
854 uint32_t i;
855
856 for (i = 0; i < oprsz; i += tysz) {
857 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
858 fni(vece, t0, t0);
859 tcg_gen_st_vec(t0, cpu_env, dofs + i);
860 }
861 tcg_temp_free_vec(t0);
862 }
863
864 /* Expand OPSZ bytes worth of two-vector operands and an immediate operand
865 using host vectors. */
866 static void expand_2i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
867 uint32_t oprsz, uint32_t tysz, TCGType type,
868 int64_t c, bool load_dest,
869 void (*fni)(unsigned, TCGv_vec, TCGv_vec, int64_t))
870 {
871 TCGv_vec t0 = tcg_temp_new_vec(type);
872 TCGv_vec t1 = tcg_temp_new_vec(type);
873 uint32_t i;
874
875 for (i = 0; i < oprsz; i += tysz) {
876 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
877 if (load_dest) {
878 tcg_gen_ld_vec(t1, cpu_env, dofs + i);
879 }
880 fni(vece, t1, t0, c);
881 tcg_gen_st_vec(t1, cpu_env, dofs + i);
882 }
883 tcg_temp_free_vec(t0);
884 tcg_temp_free_vec(t1);
885 }
886
887 static void expand_2s_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
888 uint32_t oprsz, uint32_t tysz, TCGType type,
889 TCGv_vec c, bool scalar_first,
890 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
891 {
892 TCGv_vec t0 = tcg_temp_new_vec(type);
893 TCGv_vec t1 = tcg_temp_new_vec(type);
894 uint32_t i;
895
896 for (i = 0; i < oprsz; i += tysz) {
897 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
898 if (scalar_first) {
899 fni(vece, t1, c, t0);
900 } else {
901 fni(vece, t1, t0, c);
902 }
903 tcg_gen_st_vec(t1, cpu_env, dofs + i);
904 }
905 tcg_temp_free_vec(t0);
906 tcg_temp_free_vec(t1);
907 }
908
909 /* Expand OPSZ bytes worth of three-operand operations using host vectors. */
910 static void expand_3_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
911 uint32_t bofs, uint32_t oprsz,
912 uint32_t tysz, TCGType type, bool load_dest,
913 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
914 {
915 TCGv_vec t0 = tcg_temp_new_vec(type);
916 TCGv_vec t1 = tcg_temp_new_vec(type);
917 TCGv_vec t2 = tcg_temp_new_vec(type);
918 uint32_t i;
919
920 for (i = 0; i < oprsz; i += tysz) {
921 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
922 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
923 if (load_dest) {
924 tcg_gen_ld_vec(t2, cpu_env, dofs + i);
925 }
926 fni(vece, t2, t0, t1);
927 tcg_gen_st_vec(t2, cpu_env, dofs + i);
928 }
929 tcg_temp_free_vec(t2);
930 tcg_temp_free_vec(t1);
931 tcg_temp_free_vec(t0);
932 }
933
934 /*
935 * Expand OPSZ bytes worth of three-vector operands and an immediate operand
936 * using host vectors.
937 */
938 static void expand_3i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
939 uint32_t bofs, uint32_t oprsz, uint32_t tysz,
940 TCGType type, int64_t c, bool load_dest,
941 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec,
942 int64_t))
943 {
944 TCGv_vec t0 = tcg_temp_new_vec(type);
945 TCGv_vec t1 = tcg_temp_new_vec(type);
946 TCGv_vec t2 = tcg_temp_new_vec(type);
947 uint32_t i;
948
949 for (i = 0; i < oprsz; i += tysz) {
950 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
951 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
952 if (load_dest) {
953 tcg_gen_ld_vec(t2, cpu_env, dofs + i);
954 }
955 fni(vece, t2, t0, t1, c);
956 tcg_gen_st_vec(t2, cpu_env, dofs + i);
957 }
958 tcg_temp_free_vec(t0);
959 tcg_temp_free_vec(t1);
960 tcg_temp_free_vec(t2);
961 }
962
963 /* Expand OPSZ bytes worth of four-operand operations using host vectors. */
964 static void expand_4_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
965 uint32_t bofs, uint32_t cofs, uint32_t oprsz,
966 uint32_t tysz, TCGType type, bool write_aofs,
967 void (*fni)(unsigned, TCGv_vec, TCGv_vec,
968 TCGv_vec, TCGv_vec))
969 {
970 TCGv_vec t0 = tcg_temp_new_vec(type);
971 TCGv_vec t1 = tcg_temp_new_vec(type);
972 TCGv_vec t2 = tcg_temp_new_vec(type);
973 TCGv_vec t3 = tcg_temp_new_vec(type);
974 uint32_t i;
975
976 for (i = 0; i < oprsz; i += tysz) {
977 tcg_gen_ld_vec(t1, cpu_env, aofs + i);
978 tcg_gen_ld_vec(t2, cpu_env, bofs + i);
979 tcg_gen_ld_vec(t3, cpu_env, cofs + i);
980 fni(vece, t0, t1, t2, t3);
981 tcg_gen_st_vec(t0, cpu_env, dofs + i);
982 if (write_aofs) {
983 tcg_gen_st_vec(t1, cpu_env, aofs + i);
984 }
985 }
986 tcg_temp_free_vec(t3);
987 tcg_temp_free_vec(t2);
988 tcg_temp_free_vec(t1);
989 tcg_temp_free_vec(t0);
990 }
991
992 /* Expand a vector two-operand operation. */
993 void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs,
994 uint32_t oprsz, uint32_t maxsz, const GVecGen2 *g)
995 {
996 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
997 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
998 TCGType type;
999 uint32_t some;
1000
1001 check_size_align(oprsz, maxsz, dofs | aofs);
1002 check_overlap_2(dofs, aofs, maxsz);
1003
1004 type = 0;
1005 if (g->fniv) {
1006 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1007 }
1008 switch (type) {
1009 case TCG_TYPE_V256:
1010 /* Recall that ARM SVE allows vector sizes that are not a
1011 * power of 2, but always a multiple of 16. The intent is
1012 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1013 */
1014 some = QEMU_ALIGN_DOWN(oprsz, 32);
1015 expand_2_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256, g->fniv);
1016 if (some == oprsz) {
1017 break;
1018 }
1019 dofs += some;
1020 aofs += some;
1021 oprsz -= some;
1022 maxsz -= some;
1023 /* fallthru */
1024 case TCG_TYPE_V128:
1025 expand_2_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128, g->fniv);
1026 break;
1027 case TCG_TYPE_V64:
1028 expand_2_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64, g->fniv);
1029 break;
1030
1031 case 0:
1032 if (g->fni8 && check_size_impl(oprsz, 8)) {
1033 expand_2_i64(dofs, aofs, oprsz, g->fni8);
1034 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1035 expand_2_i32(dofs, aofs, oprsz, g->fni4);
1036 } else {
1037 assert(g->fno != NULL);
1038 tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, g->data, g->fno);
1039 oprsz = maxsz;
1040 }
1041 break;
1042
1043 default:
1044 g_assert_not_reached();
1045 }
1046 tcg_swap_vecop_list(hold_list);
1047
1048 if (oprsz < maxsz) {
1049 expand_clr(dofs + oprsz, maxsz - oprsz);
1050 }
1051 }
1052
1053 /* Expand a vector operation with two vectors and an immediate. */
1054 void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1055 uint32_t maxsz, int64_t c, const GVecGen2i *g)
1056 {
1057 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1058 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1059 TCGType type;
1060 uint32_t some;
1061
1062 check_size_align(oprsz, maxsz, dofs | aofs);
1063 check_overlap_2(dofs, aofs, maxsz);
1064
1065 type = 0;
1066 if (g->fniv) {
1067 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1068 }
1069 switch (type) {
1070 case TCG_TYPE_V256:
1071 /* Recall that ARM SVE allows vector sizes that are not a
1072 * power of 2, but always a multiple of 16. The intent is
1073 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1074 */
1075 some = QEMU_ALIGN_DOWN(oprsz, 32);
1076 expand_2i_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1077 c, g->load_dest, g->fniv);
1078 if (some == oprsz) {
1079 break;
1080 }
1081 dofs += some;
1082 aofs += some;
1083 oprsz -= some;
1084 maxsz -= some;
1085 /* fallthru */
1086 case TCG_TYPE_V128:
1087 expand_2i_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1088 c, g->load_dest, g->fniv);
1089 break;
1090 case TCG_TYPE_V64:
1091 expand_2i_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1092 c, g->load_dest, g->fniv);
1093 break;
1094
1095 case 0:
1096 if (g->fni8 && check_size_impl(oprsz, 8)) {
1097 expand_2i_i64(dofs, aofs, oprsz, c, g->load_dest, g->fni8);
1098 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1099 expand_2i_i32(dofs, aofs, oprsz, c, g->load_dest, g->fni4);
1100 } else {
1101 if (g->fno) {
1102 tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, c, g->fno);
1103 } else {
1104 TCGv_i64 tcg_c = tcg_const_i64(c);
1105 tcg_gen_gvec_2i_ool(dofs, aofs, tcg_c, oprsz,
1106 maxsz, c, g->fnoi);
1107 tcg_temp_free_i64(tcg_c);
1108 }
1109 oprsz = maxsz;
1110 }
1111 break;
1112
1113 default:
1114 g_assert_not_reached();
1115 }
1116 tcg_swap_vecop_list(hold_list);
1117
1118 if (oprsz < maxsz) {
1119 expand_clr(dofs + oprsz, maxsz - oprsz);
1120 }
1121 }
1122
1123 /* Expand a vector operation with two vectors and a scalar. */
1124 void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1125 uint32_t maxsz, TCGv_i64 c, const GVecGen2s *g)
1126 {
1127 TCGType type;
1128
1129 check_size_align(oprsz, maxsz, dofs | aofs);
1130 check_overlap_2(dofs, aofs, maxsz);
1131
1132 type = 0;
1133 if (g->fniv) {
1134 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1135 }
1136 if (type != 0) {
1137 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1138 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1139 TCGv_vec t_vec = tcg_temp_new_vec(type);
1140 uint32_t some;
1141
1142 tcg_gen_dup_i64_vec(g->vece, t_vec, c);
1143
1144 switch (type) {
1145 case TCG_TYPE_V256:
1146 /* Recall that ARM SVE allows vector sizes that are not a
1147 * power of 2, but always a multiple of 16. The intent is
1148 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1149 */
1150 some = QEMU_ALIGN_DOWN(oprsz, 32);
1151 expand_2s_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1152 t_vec, g->scalar_first, g->fniv);
1153 if (some == oprsz) {
1154 break;
1155 }
1156 dofs += some;
1157 aofs += some;
1158 oprsz -= some;
1159 maxsz -= some;
1160 /* fallthru */
1161
1162 case TCG_TYPE_V128:
1163 expand_2s_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1164 t_vec, g->scalar_first, g->fniv);
1165 break;
1166
1167 case TCG_TYPE_V64:
1168 expand_2s_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1169 t_vec, g->scalar_first, g->fniv);
1170 break;
1171
1172 default:
1173 g_assert_not_reached();
1174 }
1175 tcg_temp_free_vec(t_vec);
1176 tcg_swap_vecop_list(hold_list);
1177 } else if (g->fni8 && check_size_impl(oprsz, 8)) {
1178 TCGv_i64 t64 = tcg_temp_new_i64();
1179
1180 gen_dup_i64(g->vece, t64, c);
1181 expand_2s_i64(dofs, aofs, oprsz, t64, g->scalar_first, g->fni8);
1182 tcg_temp_free_i64(t64);
1183 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1184 TCGv_i32 t32 = tcg_temp_new_i32();
1185
1186 tcg_gen_extrl_i64_i32(t32, c);
1187 gen_dup_i32(g->vece, t32, t32);
1188 expand_2s_i32(dofs, aofs, oprsz, t32, g->scalar_first, g->fni4);
1189 tcg_temp_free_i32(t32);
1190 } else {
1191 tcg_gen_gvec_2i_ool(dofs, aofs, c, oprsz, maxsz, 0, g->fno);
1192 return;
1193 }
1194
1195 if (oprsz < maxsz) {
1196 expand_clr(dofs + oprsz, maxsz - oprsz);
1197 }
1198 }
1199
1200 /* Expand a vector three-operand operation. */
1201 void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1202 uint32_t oprsz, uint32_t maxsz, const GVecGen3 *g)
1203 {
1204 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1205 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1206 TCGType type;
1207 uint32_t some;
1208
1209 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
1210 check_overlap_3(dofs, aofs, bofs, maxsz);
1211
1212 type = 0;
1213 if (g->fniv) {
1214 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1215 }
1216 switch (type) {
1217 case TCG_TYPE_V256:
1218 /* Recall that ARM SVE allows vector sizes that are not a
1219 * power of 2, but always a multiple of 16. The intent is
1220 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1221 */
1222 some = QEMU_ALIGN_DOWN(oprsz, 32);
1223 expand_3_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1224 g->load_dest, g->fniv);
1225 if (some == oprsz) {
1226 break;
1227 }
1228 dofs += some;
1229 aofs += some;
1230 bofs += some;
1231 oprsz -= some;
1232 maxsz -= some;
1233 /* fallthru */
1234 case TCG_TYPE_V128:
1235 expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1236 g->load_dest, g->fniv);
1237 break;
1238 case TCG_TYPE_V64:
1239 expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1240 g->load_dest, g->fniv);
1241 break;
1242
1243 case 0:
1244 if (g->fni8 && check_size_impl(oprsz, 8)) {
1245 expand_3_i64(dofs, aofs, bofs, oprsz, g->load_dest, g->fni8);
1246 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1247 expand_3_i32(dofs, aofs, bofs, oprsz, g->load_dest, g->fni4);
1248 } else {
1249 assert(g->fno != NULL);
1250 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz,
1251 maxsz, g->data, g->fno);
1252 oprsz = maxsz;
1253 }
1254 break;
1255
1256 default:
1257 g_assert_not_reached();
1258 }
1259 tcg_swap_vecop_list(hold_list);
1260
1261 if (oprsz < maxsz) {
1262 expand_clr(dofs + oprsz, maxsz - oprsz);
1263 }
1264 }
1265
1266 /* Expand a vector operation with three vectors and an immediate. */
1267 void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1268 uint32_t oprsz, uint32_t maxsz, int64_t c,
1269 const GVecGen3i *g)
1270 {
1271 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1272 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1273 TCGType type;
1274 uint32_t some;
1275
1276 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
1277 check_overlap_3(dofs, aofs, bofs, maxsz);
1278
1279 type = 0;
1280 if (g->fniv) {
1281 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1282 }
1283 switch (type) {
1284 case TCG_TYPE_V256:
1285 /*
1286 * Recall that ARM SVE allows vector sizes that are not a
1287 * power of 2, but always a multiple of 16. The intent is
1288 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1289 */
1290 some = QEMU_ALIGN_DOWN(oprsz, 32);
1291 expand_3i_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1292 c, g->load_dest, g->fniv);
1293 if (some == oprsz) {
1294 break;
1295 }
1296 dofs += some;
1297 aofs += some;
1298 bofs += some;
1299 oprsz -= some;
1300 maxsz -= some;
1301 /* fallthru */
1302 case TCG_TYPE_V128:
1303 expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1304 c, g->load_dest, g->fniv);
1305 break;
1306 case TCG_TYPE_V64:
1307 expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1308 c, g->load_dest, g->fniv);
1309 break;
1310
1311 case 0:
1312 if (g->fni8 && check_size_impl(oprsz, 8)) {
1313 expand_3i_i64(dofs, aofs, bofs, oprsz, c, g->load_dest, g->fni8);
1314 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1315 expand_3i_i32(dofs, aofs, bofs, oprsz, c, g->load_dest, g->fni4);
1316 } else {
1317 assert(g->fno != NULL);
1318 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, c, g->fno);
1319 oprsz = maxsz;
1320 }
1321 break;
1322
1323 default:
1324 g_assert_not_reached();
1325 }
1326 tcg_swap_vecop_list(hold_list);
1327
1328 if (oprsz < maxsz) {
1329 expand_clr(dofs + oprsz, maxsz - oprsz);
1330 }
1331 }
1332
1333 /* Expand a vector four-operand operation. */
1334 void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs,
1335 uint32_t oprsz, uint32_t maxsz, const GVecGen4 *g)
1336 {
1337 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1338 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1339 TCGType type;
1340 uint32_t some;
1341
1342 check_size_align(oprsz, maxsz, dofs | aofs | bofs | cofs);
1343 check_overlap_4(dofs, aofs, bofs, cofs, maxsz);
1344
1345 type = 0;
1346 if (g->fniv) {
1347 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1348 }
1349 switch (type) {
1350 case TCG_TYPE_V256:
1351 /* Recall that ARM SVE allows vector sizes that are not a
1352 * power of 2, but always a multiple of 16. The intent is
1353 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1354 */
1355 some = QEMU_ALIGN_DOWN(oprsz, 32);
1356 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, some,
1357 32, TCG_TYPE_V256, g->write_aofs, g->fniv);
1358 if (some == oprsz) {
1359 break;
1360 }
1361 dofs += some;
1362 aofs += some;
1363 bofs += some;
1364 cofs += some;
1365 oprsz -= some;
1366 maxsz -= some;
1367 /* fallthru */
1368 case TCG_TYPE_V128:
1369 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
1370 16, TCG_TYPE_V128, g->write_aofs, g->fniv);
1371 break;
1372 case TCG_TYPE_V64:
1373 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
1374 8, TCG_TYPE_V64, g->write_aofs, g->fniv);
1375 break;
1376
1377 case 0:
1378 if (g->fni8 && check_size_impl(oprsz, 8)) {
1379 expand_4_i64(dofs, aofs, bofs, cofs, oprsz,
1380 g->write_aofs, g->fni8);
1381 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1382 expand_4_i32(dofs, aofs, bofs, cofs, oprsz,
1383 g->write_aofs, g->fni4);
1384 } else {
1385 assert(g->fno != NULL);
1386 tcg_gen_gvec_4_ool(dofs, aofs, bofs, cofs,
1387 oprsz, maxsz, g->data, g->fno);
1388 oprsz = maxsz;
1389 }
1390 break;
1391
1392 default:
1393 g_assert_not_reached();
1394 }
1395 tcg_swap_vecop_list(hold_list);
1396
1397 if (oprsz < maxsz) {
1398 expand_clr(dofs + oprsz, maxsz - oprsz);
1399 }
1400 }
1401
1402 /*
1403 * Expand specific vector operations.
1404 */
1405
1406 static void vec_mov2(unsigned vece, TCGv_vec a, TCGv_vec b)
1407 {
1408 tcg_gen_mov_vec(a, b);
1409 }
1410
1411 void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs,
1412 uint32_t oprsz, uint32_t maxsz)
1413 {
1414 static const GVecGen2 g = {
1415 .fni8 = tcg_gen_mov_i64,
1416 .fniv = vec_mov2,
1417 .fno = gen_helper_gvec_mov,
1418 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1419 };
1420 if (dofs != aofs) {
1421 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1422 } else {
1423 check_size_align(oprsz, maxsz, dofs);
1424 if (oprsz < maxsz) {
1425 expand_clr(dofs + oprsz, maxsz - oprsz);
1426 }
1427 }
1428 }
1429
1430 void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t oprsz,
1431 uint32_t maxsz, TCGv_i32 in)
1432 {
1433 check_size_align(oprsz, maxsz, dofs);
1434 tcg_debug_assert(vece <= MO_32);
1435 do_dup(vece, dofs, oprsz, maxsz, in, NULL, 0);
1436 }
1437
1438 void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t oprsz,
1439 uint32_t maxsz, TCGv_i64 in)
1440 {
1441 check_size_align(oprsz, maxsz, dofs);
1442 tcg_debug_assert(vece <= MO_64);
1443 do_dup(vece, dofs, oprsz, maxsz, NULL, in, 0);
1444 }
1445
1446 void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs,
1447 uint32_t oprsz, uint32_t maxsz)
1448 {
1449 check_size_align(oprsz, maxsz, dofs);
1450 if (vece <= MO_64) {
1451 TCGType type = choose_vector_type(NULL, vece, oprsz, 0);
1452 if (type != 0) {
1453 TCGv_vec t_vec = tcg_temp_new_vec(type);
1454 tcg_gen_dup_mem_vec(vece, t_vec, cpu_env, aofs);
1455 do_dup_store(type, dofs, oprsz, maxsz, t_vec);
1456 tcg_temp_free_vec(t_vec);
1457 } else if (vece <= MO_32) {
1458 TCGv_i32 in = tcg_temp_new_i32();
1459 switch (vece) {
1460 case MO_8:
1461 tcg_gen_ld8u_i32(in, cpu_env, aofs);
1462 break;
1463 case MO_16:
1464 tcg_gen_ld16u_i32(in, cpu_env, aofs);
1465 break;
1466 default:
1467 tcg_gen_ld_i32(in, cpu_env, aofs);
1468 break;
1469 }
1470 do_dup(vece, dofs, oprsz, maxsz, in, NULL, 0);
1471 tcg_temp_free_i32(in);
1472 } else {
1473 TCGv_i64 in = tcg_temp_new_i64();
1474 tcg_gen_ld_i64(in, cpu_env, aofs);
1475 do_dup(vece, dofs, oprsz, maxsz, NULL, in, 0);
1476 tcg_temp_free_i64(in);
1477 }
1478 } else {
1479 /* 128-bit duplicate. */
1480 /* ??? Dup to 256-bit vector. */
1481 int i;
1482
1483 tcg_debug_assert(vece == 4);
1484 tcg_debug_assert(oprsz >= 16);
1485 if (TCG_TARGET_HAS_v128) {
1486 TCGv_vec in = tcg_temp_new_vec(TCG_TYPE_V128);
1487
1488 tcg_gen_ld_vec(in, cpu_env, aofs);
1489 for (i = 0; i < oprsz; i += 16) {
1490 tcg_gen_st_vec(in, cpu_env, dofs + i);
1491 }
1492 tcg_temp_free_vec(in);
1493 } else {
1494 TCGv_i64 in0 = tcg_temp_new_i64();
1495 TCGv_i64 in1 = tcg_temp_new_i64();
1496
1497 tcg_gen_ld_i64(in0, cpu_env, aofs);
1498 tcg_gen_ld_i64(in1, cpu_env, aofs + 8);
1499 for (i = 0; i < oprsz; i += 16) {
1500 tcg_gen_st_i64(in0, cpu_env, dofs + i);
1501 tcg_gen_st_i64(in1, cpu_env, dofs + i + 8);
1502 }
1503 tcg_temp_free_i64(in0);
1504 tcg_temp_free_i64(in1);
1505 }
1506 if (oprsz < maxsz) {
1507 expand_clr(dofs + oprsz, maxsz - oprsz);
1508 }
1509 }
1510 }
1511
1512 void tcg_gen_gvec_dup64i(uint32_t dofs, uint32_t oprsz,
1513 uint32_t maxsz, uint64_t x)
1514 {
1515 check_size_align(oprsz, maxsz, dofs);
1516 do_dup(MO_64, dofs, oprsz, maxsz, NULL, NULL, x);
1517 }
1518
1519 void tcg_gen_gvec_dup32i(uint32_t dofs, uint32_t oprsz,
1520 uint32_t maxsz, uint32_t x)
1521 {
1522 check_size_align(oprsz, maxsz, dofs);
1523 do_dup(MO_32, dofs, oprsz, maxsz, NULL, NULL, x);
1524 }
1525
1526 void tcg_gen_gvec_dup16i(uint32_t dofs, uint32_t oprsz,
1527 uint32_t maxsz, uint16_t x)
1528 {
1529 check_size_align(oprsz, maxsz, dofs);
1530 do_dup(MO_16, dofs, oprsz, maxsz, NULL, NULL, x);
1531 }
1532
1533 void tcg_gen_gvec_dup8i(uint32_t dofs, uint32_t oprsz,
1534 uint32_t maxsz, uint8_t x)
1535 {
1536 check_size_align(oprsz, maxsz, dofs);
1537 do_dup(MO_8, dofs, oprsz, maxsz, NULL, NULL, x);
1538 }
1539
1540 void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs,
1541 uint32_t oprsz, uint32_t maxsz)
1542 {
1543 static const GVecGen2 g = {
1544 .fni8 = tcg_gen_not_i64,
1545 .fniv = tcg_gen_not_vec,
1546 .fno = gen_helper_gvec_not,
1547 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1548 };
1549 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1550 }
1551
1552 /* Perform a vector addition using normal addition and a mask. The mask
1553 should be the sign bit of each lane. This 6-operation form is more
1554 efficient than separate additions when there are 4 or more lanes in
1555 the 64-bit operation. */
1556 static void gen_addv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1557 {
1558 TCGv_i64 t1 = tcg_temp_new_i64();
1559 TCGv_i64 t2 = tcg_temp_new_i64();
1560 TCGv_i64 t3 = tcg_temp_new_i64();
1561
1562 tcg_gen_andc_i64(t1, a, m);
1563 tcg_gen_andc_i64(t2, b, m);
1564 tcg_gen_xor_i64(t3, a, b);
1565 tcg_gen_add_i64(d, t1, t2);
1566 tcg_gen_and_i64(t3, t3, m);
1567 tcg_gen_xor_i64(d, d, t3);
1568
1569 tcg_temp_free_i64(t1);
1570 tcg_temp_free_i64(t2);
1571 tcg_temp_free_i64(t3);
1572 }
1573
1574 void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1575 {
1576 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
1577 gen_addv_mask(d, a, b, m);
1578 tcg_temp_free_i64(m);
1579 }
1580
1581 void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1582 {
1583 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
1584 gen_addv_mask(d, a, b, m);
1585 tcg_temp_free_i64(m);
1586 }
1587
1588 void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1589 {
1590 TCGv_i64 t1 = tcg_temp_new_i64();
1591 TCGv_i64 t2 = tcg_temp_new_i64();
1592
1593 tcg_gen_andi_i64(t1, a, ~0xffffffffull);
1594 tcg_gen_add_i64(t2, a, b);
1595 tcg_gen_add_i64(t1, t1, b);
1596 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
1597
1598 tcg_temp_free_i64(t1);
1599 tcg_temp_free_i64(t2);
1600 }
1601
1602 static const TCGOpcode vecop_list_add[] = { INDEX_op_add_vec, 0 };
1603
1604 void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs,
1605 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1606 {
1607 static const GVecGen3 g[4] = {
1608 { .fni8 = tcg_gen_vec_add8_i64,
1609 .fniv = tcg_gen_add_vec,
1610 .fno = gen_helper_gvec_add8,
1611 .opt_opc = vecop_list_add,
1612 .vece = MO_8 },
1613 { .fni8 = tcg_gen_vec_add16_i64,
1614 .fniv = tcg_gen_add_vec,
1615 .fno = gen_helper_gvec_add16,
1616 .opt_opc = vecop_list_add,
1617 .vece = MO_16 },
1618 { .fni4 = tcg_gen_add_i32,
1619 .fniv = tcg_gen_add_vec,
1620 .fno = gen_helper_gvec_add32,
1621 .opt_opc = vecop_list_add,
1622 .vece = MO_32 },
1623 { .fni8 = tcg_gen_add_i64,
1624 .fniv = tcg_gen_add_vec,
1625 .fno = gen_helper_gvec_add64,
1626 .opt_opc = vecop_list_add,
1627 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1628 .vece = MO_64 },
1629 };
1630
1631 tcg_debug_assert(vece <= MO_64);
1632 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1633 }
1634
1635 void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs,
1636 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1637 {
1638 static const GVecGen2s g[4] = {
1639 { .fni8 = tcg_gen_vec_add8_i64,
1640 .fniv = tcg_gen_add_vec,
1641 .fno = gen_helper_gvec_adds8,
1642 .opt_opc = vecop_list_add,
1643 .vece = MO_8 },
1644 { .fni8 = tcg_gen_vec_add16_i64,
1645 .fniv = tcg_gen_add_vec,
1646 .fno = gen_helper_gvec_adds16,
1647 .opt_opc = vecop_list_add,
1648 .vece = MO_16 },
1649 { .fni4 = tcg_gen_add_i32,
1650 .fniv = tcg_gen_add_vec,
1651 .fno = gen_helper_gvec_adds32,
1652 .opt_opc = vecop_list_add,
1653 .vece = MO_32 },
1654 { .fni8 = tcg_gen_add_i64,
1655 .fniv = tcg_gen_add_vec,
1656 .fno = gen_helper_gvec_adds64,
1657 .opt_opc = vecop_list_add,
1658 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1659 .vece = MO_64 },
1660 };
1661
1662 tcg_debug_assert(vece <= MO_64);
1663 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1664 }
1665
1666 void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs,
1667 int64_t c, uint32_t oprsz, uint32_t maxsz)
1668 {
1669 TCGv_i64 tmp = tcg_const_i64(c);
1670 tcg_gen_gvec_adds(vece, dofs, aofs, tmp, oprsz, maxsz);
1671 tcg_temp_free_i64(tmp);
1672 }
1673
1674 static const TCGOpcode vecop_list_sub[] = { INDEX_op_sub_vec, 0 };
1675
1676 void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs,
1677 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1678 {
1679 static const GVecGen2s g[4] = {
1680 { .fni8 = tcg_gen_vec_sub8_i64,
1681 .fniv = tcg_gen_sub_vec,
1682 .fno = gen_helper_gvec_subs8,
1683 .opt_opc = vecop_list_sub,
1684 .vece = MO_8 },
1685 { .fni8 = tcg_gen_vec_sub16_i64,
1686 .fniv = tcg_gen_sub_vec,
1687 .fno = gen_helper_gvec_subs16,
1688 .opt_opc = vecop_list_sub,
1689 .vece = MO_16 },
1690 { .fni4 = tcg_gen_sub_i32,
1691 .fniv = tcg_gen_sub_vec,
1692 .fno = gen_helper_gvec_subs32,
1693 .opt_opc = vecop_list_sub,
1694 .vece = MO_32 },
1695 { .fni8 = tcg_gen_sub_i64,
1696 .fniv = tcg_gen_sub_vec,
1697 .fno = gen_helper_gvec_subs64,
1698 .opt_opc = vecop_list_sub,
1699 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1700 .vece = MO_64 },
1701 };
1702
1703 tcg_debug_assert(vece <= MO_64);
1704 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1705 }
1706
1707 /* Perform a vector subtraction using normal subtraction and a mask.
1708 Compare gen_addv_mask above. */
1709 static void gen_subv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1710 {
1711 TCGv_i64 t1 = tcg_temp_new_i64();
1712 TCGv_i64 t2 = tcg_temp_new_i64();
1713 TCGv_i64 t3 = tcg_temp_new_i64();
1714
1715 tcg_gen_or_i64(t1, a, m);
1716 tcg_gen_andc_i64(t2, b, m);
1717 tcg_gen_eqv_i64(t3, a, b);
1718 tcg_gen_sub_i64(d, t1, t2);
1719 tcg_gen_and_i64(t3, t3, m);
1720 tcg_gen_xor_i64(d, d, t3);
1721
1722 tcg_temp_free_i64(t1);
1723 tcg_temp_free_i64(t2);
1724 tcg_temp_free_i64(t3);
1725 }
1726
1727 void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1728 {
1729 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
1730 gen_subv_mask(d, a, b, m);
1731 tcg_temp_free_i64(m);
1732 }
1733
1734 void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1735 {
1736 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
1737 gen_subv_mask(d, a, b, m);
1738 tcg_temp_free_i64(m);
1739 }
1740
1741 void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1742 {
1743 TCGv_i64 t1 = tcg_temp_new_i64();
1744 TCGv_i64 t2 = tcg_temp_new_i64();
1745
1746 tcg_gen_andi_i64(t1, b, ~0xffffffffull);
1747 tcg_gen_sub_i64(t2, a, b);
1748 tcg_gen_sub_i64(t1, a, t1);
1749 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
1750
1751 tcg_temp_free_i64(t1);
1752 tcg_temp_free_i64(t2);
1753 }
1754
1755 void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs,
1756 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1757 {
1758 static const GVecGen3 g[4] = {
1759 { .fni8 = tcg_gen_vec_sub8_i64,
1760 .fniv = tcg_gen_sub_vec,
1761 .fno = gen_helper_gvec_sub8,
1762 .opt_opc = vecop_list_sub,
1763 .vece = MO_8 },
1764 { .fni8 = tcg_gen_vec_sub16_i64,
1765 .fniv = tcg_gen_sub_vec,
1766 .fno = gen_helper_gvec_sub16,
1767 .opt_opc = vecop_list_sub,
1768 .vece = MO_16 },
1769 { .fni4 = tcg_gen_sub_i32,
1770 .fniv = tcg_gen_sub_vec,
1771 .fno = gen_helper_gvec_sub32,
1772 .opt_opc = vecop_list_sub,
1773 .vece = MO_32 },
1774 { .fni8 = tcg_gen_sub_i64,
1775 .fniv = tcg_gen_sub_vec,
1776 .fno = gen_helper_gvec_sub64,
1777 .opt_opc = vecop_list_sub,
1778 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1779 .vece = MO_64 },
1780 };
1781
1782 tcg_debug_assert(vece <= MO_64);
1783 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1784 }
1785
1786 static const TCGOpcode vecop_list_mul[] = { INDEX_op_mul_vec, 0 };
1787
1788 void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs,
1789 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1790 {
1791 static const GVecGen3 g[4] = {
1792 { .fniv = tcg_gen_mul_vec,
1793 .fno = gen_helper_gvec_mul8,
1794 .opt_opc = vecop_list_mul,
1795 .vece = MO_8 },
1796 { .fniv = tcg_gen_mul_vec,
1797 .fno = gen_helper_gvec_mul16,
1798 .opt_opc = vecop_list_mul,
1799 .vece = MO_16 },
1800 { .fni4 = tcg_gen_mul_i32,
1801 .fniv = tcg_gen_mul_vec,
1802 .fno = gen_helper_gvec_mul32,
1803 .opt_opc = vecop_list_mul,
1804 .vece = MO_32 },
1805 { .fni8 = tcg_gen_mul_i64,
1806 .fniv = tcg_gen_mul_vec,
1807 .fno = gen_helper_gvec_mul64,
1808 .opt_opc = vecop_list_mul,
1809 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1810 .vece = MO_64 },
1811 };
1812
1813 tcg_debug_assert(vece <= MO_64);
1814 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1815 }
1816
1817 void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs,
1818 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1819 {
1820 static const GVecGen2s g[4] = {
1821 { .fniv = tcg_gen_mul_vec,
1822 .fno = gen_helper_gvec_muls8,
1823 .opt_opc = vecop_list_mul,
1824 .vece = MO_8 },
1825 { .fniv = tcg_gen_mul_vec,
1826 .fno = gen_helper_gvec_muls16,
1827 .opt_opc = vecop_list_mul,
1828 .vece = MO_16 },
1829 { .fni4 = tcg_gen_mul_i32,
1830 .fniv = tcg_gen_mul_vec,
1831 .fno = gen_helper_gvec_muls32,
1832 .opt_opc = vecop_list_mul,
1833 .vece = MO_32 },
1834 { .fni8 = tcg_gen_mul_i64,
1835 .fniv = tcg_gen_mul_vec,
1836 .fno = gen_helper_gvec_muls64,
1837 .opt_opc = vecop_list_mul,
1838 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1839 .vece = MO_64 },
1840 };
1841
1842 tcg_debug_assert(vece <= MO_64);
1843 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1844 }
1845
1846 void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs,
1847 int64_t c, uint32_t oprsz, uint32_t maxsz)
1848 {
1849 TCGv_i64 tmp = tcg_const_i64(c);
1850 tcg_gen_gvec_muls(vece, dofs, aofs, tmp, oprsz, maxsz);
1851 tcg_temp_free_i64(tmp);
1852 }
1853
1854 void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs,
1855 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1856 {
1857 static const TCGOpcode vecop_list[] = { INDEX_op_ssadd_vec, 0 };
1858 static const GVecGen3 g[4] = {
1859 { .fniv = tcg_gen_ssadd_vec,
1860 .fno = gen_helper_gvec_ssadd8,
1861 .opt_opc = vecop_list,
1862 .vece = MO_8 },
1863 { .fniv = tcg_gen_ssadd_vec,
1864 .fno = gen_helper_gvec_ssadd16,
1865 .opt_opc = vecop_list,
1866 .vece = MO_16 },
1867 { .fniv = tcg_gen_ssadd_vec,
1868 .fno = gen_helper_gvec_ssadd32,
1869 .opt_opc = vecop_list,
1870 .vece = MO_32 },
1871 { .fniv = tcg_gen_ssadd_vec,
1872 .fno = gen_helper_gvec_ssadd64,
1873 .opt_opc = vecop_list,
1874 .vece = MO_64 },
1875 };
1876 tcg_debug_assert(vece <= MO_64);
1877 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1878 }
1879
1880 void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs,
1881 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1882 {
1883 static const TCGOpcode vecop_list[] = { INDEX_op_sssub_vec, 0 };
1884 static const GVecGen3 g[4] = {
1885 { .fniv = tcg_gen_sssub_vec,
1886 .fno = gen_helper_gvec_sssub8,
1887 .opt_opc = vecop_list,
1888 .vece = MO_8 },
1889 { .fniv = tcg_gen_sssub_vec,
1890 .fno = gen_helper_gvec_sssub16,
1891 .opt_opc = vecop_list,
1892 .vece = MO_16 },
1893 { .fniv = tcg_gen_sssub_vec,
1894 .fno = gen_helper_gvec_sssub32,
1895 .opt_opc = vecop_list,
1896 .vece = MO_32 },
1897 { .fniv = tcg_gen_sssub_vec,
1898 .fno = gen_helper_gvec_sssub64,
1899 .opt_opc = vecop_list,
1900 .vece = MO_64 },
1901 };
1902 tcg_debug_assert(vece <= MO_64);
1903 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1904 }
1905
1906 static void tcg_gen_usadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1907 {
1908 TCGv_i32 max = tcg_const_i32(-1);
1909 tcg_gen_add_i32(d, a, b);
1910 tcg_gen_movcond_i32(TCG_COND_LTU, d, d, a, max, d);
1911 tcg_temp_free_i32(max);
1912 }
1913
1914 static void tcg_gen_usadd_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1915 {
1916 TCGv_i64 max = tcg_const_i64(-1);
1917 tcg_gen_add_i64(d, a, b);
1918 tcg_gen_movcond_i64(TCG_COND_LTU, d, d, a, max, d);
1919 tcg_temp_free_i64(max);
1920 }
1921
1922 void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs,
1923 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1924 {
1925 static const TCGOpcode vecop_list[] = { INDEX_op_usadd_vec, 0 };
1926 static const GVecGen3 g[4] = {
1927 { .fniv = tcg_gen_usadd_vec,
1928 .fno = gen_helper_gvec_usadd8,
1929 .opt_opc = vecop_list,
1930 .vece = MO_8 },
1931 { .fniv = tcg_gen_usadd_vec,
1932 .fno = gen_helper_gvec_usadd16,
1933 .opt_opc = vecop_list,
1934 .vece = MO_16 },
1935 { .fni4 = tcg_gen_usadd_i32,
1936 .fniv = tcg_gen_usadd_vec,
1937 .fno = gen_helper_gvec_usadd32,
1938 .opt_opc = vecop_list,
1939 .vece = MO_32 },
1940 { .fni8 = tcg_gen_usadd_i64,
1941 .fniv = tcg_gen_usadd_vec,
1942 .fno = gen_helper_gvec_usadd64,
1943 .opt_opc = vecop_list,
1944 .vece = MO_64 }
1945 };
1946 tcg_debug_assert(vece <= MO_64);
1947 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1948 }
1949
1950 static void tcg_gen_ussub_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1951 {
1952 TCGv_i32 min = tcg_const_i32(0);
1953 tcg_gen_sub_i32(d, a, b);
1954 tcg_gen_movcond_i32(TCG_COND_LTU, d, a, b, min, d);
1955 tcg_temp_free_i32(min);
1956 }
1957
1958 static void tcg_gen_ussub_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1959 {
1960 TCGv_i64 min = tcg_const_i64(0);
1961 tcg_gen_sub_i64(d, a, b);
1962 tcg_gen_movcond_i64(TCG_COND_LTU, d, a, b, min, d);
1963 tcg_temp_free_i64(min);
1964 }
1965
1966 void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs,
1967 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1968 {
1969 static const TCGOpcode vecop_list[] = { INDEX_op_ussub_vec, 0 };
1970 static const GVecGen3 g[4] = {
1971 { .fniv = tcg_gen_ussub_vec,
1972 .fno = gen_helper_gvec_ussub8,
1973 .opt_opc = vecop_list,
1974 .vece = MO_8 },
1975 { .fniv = tcg_gen_ussub_vec,
1976 .fno = gen_helper_gvec_ussub16,
1977 .opt_opc = vecop_list,
1978 .vece = MO_16 },
1979 { .fni4 = tcg_gen_ussub_i32,
1980 .fniv = tcg_gen_ussub_vec,
1981 .fno = gen_helper_gvec_ussub32,
1982 .opt_opc = vecop_list,
1983 .vece = MO_32 },
1984 { .fni8 = tcg_gen_ussub_i64,
1985 .fniv = tcg_gen_ussub_vec,
1986 .fno = gen_helper_gvec_ussub64,
1987 .opt_opc = vecop_list,
1988 .vece = MO_64 }
1989 };
1990 tcg_debug_assert(vece <= MO_64);
1991 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1992 }
1993
1994 void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs,
1995 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1996 {
1997 static const TCGOpcode vecop_list[] = { INDEX_op_smin_vec, 0 };
1998 static const GVecGen3 g[4] = {
1999 { .fniv = tcg_gen_smin_vec,
2000 .fno = gen_helper_gvec_smin8,
2001 .opt_opc = vecop_list,
2002 .vece = MO_8 },
2003 { .fniv = tcg_gen_smin_vec,
2004 .fno = gen_helper_gvec_smin16,
2005 .opt_opc = vecop_list,
2006 .vece = MO_16 },
2007 { .fni4 = tcg_gen_smin_i32,
2008 .fniv = tcg_gen_smin_vec,
2009 .fno = gen_helper_gvec_smin32,
2010 .opt_opc = vecop_list,
2011 .vece = MO_32 },
2012 { .fni8 = tcg_gen_smin_i64,
2013 .fniv = tcg_gen_smin_vec,
2014 .fno = gen_helper_gvec_smin64,
2015 .opt_opc = vecop_list,
2016 .vece = MO_64 }
2017 };
2018 tcg_debug_assert(vece <= MO_64);
2019 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2020 }
2021
2022 void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs,
2023 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2024 {
2025 static const TCGOpcode vecop_list[] = { INDEX_op_umin_vec, 0 };
2026 static const GVecGen3 g[4] = {
2027 { .fniv = tcg_gen_umin_vec,
2028 .fno = gen_helper_gvec_umin8,
2029 .opt_opc = vecop_list,
2030 .vece = MO_8 },
2031 { .fniv = tcg_gen_umin_vec,
2032 .fno = gen_helper_gvec_umin16,
2033 .opt_opc = vecop_list,
2034 .vece = MO_16 },
2035 { .fni4 = tcg_gen_umin_i32,
2036 .fniv = tcg_gen_umin_vec,
2037 .fno = gen_helper_gvec_umin32,
2038 .opt_opc = vecop_list,
2039 .vece = MO_32 },
2040 { .fni8 = tcg_gen_umin_i64,
2041 .fniv = tcg_gen_umin_vec,
2042 .fno = gen_helper_gvec_umin64,
2043 .opt_opc = vecop_list,
2044 .vece = MO_64 }
2045 };
2046 tcg_debug_assert(vece <= MO_64);
2047 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2048 }
2049
2050 void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs,
2051 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2052 {
2053 static const TCGOpcode vecop_list[] = { INDEX_op_smax_vec, 0 };
2054 static const GVecGen3 g[4] = {
2055 { .fniv = tcg_gen_smax_vec,
2056 .fno = gen_helper_gvec_smax8,
2057 .opt_opc = vecop_list,
2058 .vece = MO_8 },
2059 { .fniv = tcg_gen_smax_vec,
2060 .fno = gen_helper_gvec_smax16,
2061 .opt_opc = vecop_list,
2062 .vece = MO_16 },
2063 { .fni4 = tcg_gen_smax_i32,
2064 .fniv = tcg_gen_smax_vec,
2065 .fno = gen_helper_gvec_smax32,
2066 .opt_opc = vecop_list,
2067 .vece = MO_32 },
2068 { .fni8 = tcg_gen_smax_i64,
2069 .fniv = tcg_gen_smax_vec,
2070 .fno = gen_helper_gvec_smax64,
2071 .opt_opc = vecop_list,
2072 .vece = MO_64 }
2073 };
2074 tcg_debug_assert(vece <= MO_64);
2075 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2076 }
2077
2078 void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs,
2079 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2080 {
2081 static const TCGOpcode vecop_list[] = { INDEX_op_umax_vec, 0 };
2082 static const GVecGen3 g[4] = {
2083 { .fniv = tcg_gen_umax_vec,
2084 .fno = gen_helper_gvec_umax8,
2085 .opt_opc = vecop_list,
2086 .vece = MO_8 },
2087 { .fniv = tcg_gen_umax_vec,
2088 .fno = gen_helper_gvec_umax16,
2089 .opt_opc = vecop_list,
2090 .vece = MO_16 },
2091 { .fni4 = tcg_gen_umax_i32,
2092 .fniv = tcg_gen_umax_vec,
2093 .fno = gen_helper_gvec_umax32,
2094 .opt_opc = vecop_list,
2095 .vece = MO_32 },
2096 { .fni8 = tcg_gen_umax_i64,
2097 .fniv = tcg_gen_umax_vec,
2098 .fno = gen_helper_gvec_umax64,
2099 .opt_opc = vecop_list,
2100 .vece = MO_64 }
2101 };
2102 tcg_debug_assert(vece <= MO_64);
2103 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2104 }
2105
2106 /* Perform a vector negation using normal negation and a mask.
2107 Compare gen_subv_mask above. */
2108 static void gen_negv_mask(TCGv_i64 d, TCGv_i64 b, TCGv_i64 m)
2109 {
2110 TCGv_i64 t2 = tcg_temp_new_i64();
2111 TCGv_i64 t3 = tcg_temp_new_i64();
2112
2113 tcg_gen_andc_i64(t3, m, b);
2114 tcg_gen_andc_i64(t2, b, m);
2115 tcg_gen_sub_i64(d, m, t2);
2116 tcg_gen_xor_i64(d, d, t3);
2117
2118 tcg_temp_free_i64(t2);
2119 tcg_temp_free_i64(t3);
2120 }
2121
2122 void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 b)
2123 {
2124 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
2125 gen_negv_mask(d, b, m);
2126 tcg_temp_free_i64(m);
2127 }
2128
2129 void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 b)
2130 {
2131 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
2132 gen_negv_mask(d, b, m);
2133 tcg_temp_free_i64(m);
2134 }
2135
2136 void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 b)
2137 {
2138 TCGv_i64 t1 = tcg_temp_new_i64();
2139 TCGv_i64 t2 = tcg_temp_new_i64();
2140
2141 tcg_gen_andi_i64(t1, b, ~0xffffffffull);
2142 tcg_gen_neg_i64(t2, b);
2143 tcg_gen_neg_i64(t1, t1);
2144 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
2145
2146 tcg_temp_free_i64(t1);
2147 tcg_temp_free_i64(t2);
2148 }
2149
2150 void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs,
2151 uint32_t oprsz, uint32_t maxsz)
2152 {
2153 static const TCGOpcode vecop_list[] = { INDEX_op_neg_vec, 0 };
2154 static const GVecGen2 g[4] = {
2155 { .fni8 = tcg_gen_vec_neg8_i64,
2156 .fniv = tcg_gen_neg_vec,
2157 .fno = gen_helper_gvec_neg8,
2158 .opt_opc = vecop_list,
2159 .vece = MO_8 },
2160 { .fni8 = tcg_gen_vec_neg16_i64,
2161 .fniv = tcg_gen_neg_vec,
2162 .fno = gen_helper_gvec_neg16,
2163 .opt_opc = vecop_list,
2164 .vece = MO_16 },
2165 { .fni4 = tcg_gen_neg_i32,
2166 .fniv = tcg_gen_neg_vec,
2167 .fno = gen_helper_gvec_neg32,
2168 .opt_opc = vecop_list,
2169 .vece = MO_32 },
2170 { .fni8 = tcg_gen_neg_i64,
2171 .fniv = tcg_gen_neg_vec,
2172 .fno = gen_helper_gvec_neg64,
2173 .opt_opc = vecop_list,
2174 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2175 .vece = MO_64 },
2176 };
2177
2178 tcg_debug_assert(vece <= MO_64);
2179 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2180 }
2181
2182 static void gen_absv_mask(TCGv_i64 d, TCGv_i64 b, unsigned vece)
2183 {
2184 TCGv_i64 t = tcg_temp_new_i64();
2185 int nbit = 8 << vece;
2186
2187 /* Create -1 for each negative element. */
2188 tcg_gen_shri_i64(t, b, nbit - 1);
2189 tcg_gen_andi_i64(t, t, dup_const(vece, 1));
2190 tcg_gen_muli_i64(t, t, (1 << nbit) - 1);
2191
2192 /*
2193 * Invert (via xor -1) and add one (via sub -1).
2194 * Because of the ordering the msb is cleared,
2195 * so we never have carry into the next element.
2196 */
2197 tcg_gen_xor_i64(d, b, t);
2198 tcg_gen_sub_i64(d, d, t);
2199
2200 tcg_temp_free_i64(t);
2201 }
2202
2203 static void tcg_gen_vec_abs8_i64(TCGv_i64 d, TCGv_i64 b)
2204 {
2205 gen_absv_mask(d, b, MO_8);
2206 }
2207
2208 static void tcg_gen_vec_abs16_i64(TCGv_i64 d, TCGv_i64 b)
2209 {
2210 gen_absv_mask(d, b, MO_16);
2211 }
2212
2213 void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs,
2214 uint32_t oprsz, uint32_t maxsz)
2215 {
2216 static const TCGOpcode vecop_list[] = { INDEX_op_abs_vec, 0 };
2217 static const GVecGen2 g[4] = {
2218 { .fni8 = tcg_gen_vec_abs8_i64,
2219 .fniv = tcg_gen_abs_vec,
2220 .fno = gen_helper_gvec_abs8,
2221 .opt_opc = vecop_list,
2222 .vece = MO_8 },
2223 { .fni8 = tcg_gen_vec_abs16_i64,
2224 .fniv = tcg_gen_abs_vec,
2225 .fno = gen_helper_gvec_abs16,
2226 .opt_opc = vecop_list,
2227 .vece = MO_16 },
2228 { .fni4 = tcg_gen_abs_i32,
2229 .fniv = tcg_gen_abs_vec,
2230 .fno = gen_helper_gvec_abs32,
2231 .opt_opc = vecop_list,
2232 .vece = MO_32 },
2233 { .fni8 = tcg_gen_abs_i64,
2234 .fniv = tcg_gen_abs_vec,
2235 .fno = gen_helper_gvec_abs64,
2236 .opt_opc = vecop_list,
2237 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2238 .vece = MO_64 },
2239 };
2240
2241 tcg_debug_assert(vece <= MO_64);
2242 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2243 }
2244
2245 void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs,
2246 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2247 {
2248 static const GVecGen3 g = {
2249 .fni8 = tcg_gen_and_i64,
2250 .fniv = tcg_gen_and_vec,
2251 .fno = gen_helper_gvec_and,
2252 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2253 };
2254
2255 if (aofs == bofs) {
2256 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2257 } else {
2258 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2259 }
2260 }
2261
2262 void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs,
2263 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2264 {
2265 static const GVecGen3 g = {
2266 .fni8 = tcg_gen_or_i64,
2267 .fniv = tcg_gen_or_vec,
2268 .fno = gen_helper_gvec_or,
2269 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2270 };
2271
2272 if (aofs == bofs) {
2273 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2274 } else {
2275 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2276 }
2277 }
2278
2279 void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs,
2280 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2281 {
2282 static const GVecGen3 g = {
2283 .fni8 = tcg_gen_xor_i64,
2284 .fniv = tcg_gen_xor_vec,
2285 .fno = gen_helper_gvec_xor,
2286 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2287 };
2288
2289 if (aofs == bofs) {
2290 tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, 0);
2291 } else {
2292 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2293 }
2294 }
2295
2296 void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs,
2297 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2298 {
2299 static const GVecGen3 g = {
2300 .fni8 = tcg_gen_andc_i64,
2301 .fniv = tcg_gen_andc_vec,
2302 .fno = gen_helper_gvec_andc,
2303 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2304 };
2305
2306 if (aofs == bofs) {
2307 tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, 0);
2308 } else {
2309 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2310 }
2311 }
2312
2313 void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs,
2314 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2315 {
2316 static const GVecGen3 g = {
2317 .fni8 = tcg_gen_orc_i64,
2318 .fniv = tcg_gen_orc_vec,
2319 .fno = gen_helper_gvec_orc,
2320 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2321 };
2322
2323 if (aofs == bofs) {
2324 tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, -1);
2325 } else {
2326 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2327 }
2328 }
2329
2330 void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs,
2331 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2332 {
2333 static const GVecGen3 g = {
2334 .fni8 = tcg_gen_nand_i64,
2335 .fniv = tcg_gen_nand_vec,
2336 .fno = gen_helper_gvec_nand,
2337 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2338 };
2339
2340 if (aofs == bofs) {
2341 tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2342 } else {
2343 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2344 }
2345 }
2346
2347 void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs,
2348 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2349 {
2350 static const GVecGen3 g = {
2351 .fni8 = tcg_gen_nor_i64,
2352 .fniv = tcg_gen_nor_vec,
2353 .fno = gen_helper_gvec_nor,
2354 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2355 };
2356
2357 if (aofs == bofs) {
2358 tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2359 } else {
2360 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2361 }
2362 }
2363
2364 void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs,
2365 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2366 {
2367 static const GVecGen3 g = {
2368 .fni8 = tcg_gen_eqv_i64,
2369 .fniv = tcg_gen_eqv_vec,
2370 .fno = gen_helper_gvec_eqv,
2371 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2372 };
2373
2374 if (aofs == bofs) {
2375 tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, -1);
2376 } else {
2377 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2378 }
2379 }
2380
2381 static const GVecGen2s gop_ands = {
2382 .fni8 = tcg_gen_and_i64,
2383 .fniv = tcg_gen_and_vec,
2384 .fno = gen_helper_gvec_ands,
2385 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2386 .vece = MO_64
2387 };
2388
2389 void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs,
2390 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2391 {
2392 TCGv_i64 tmp = tcg_temp_new_i64();
2393 gen_dup_i64(vece, tmp, c);
2394 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2395 tcg_temp_free_i64(tmp);
2396 }
2397
2398 void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs,
2399 int64_t c, uint32_t oprsz, uint32_t maxsz)
2400 {
2401 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2402 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2403 tcg_temp_free_i64(tmp);
2404 }
2405
2406 static const GVecGen2s gop_xors = {
2407 .fni8 = tcg_gen_xor_i64,
2408 .fniv = tcg_gen_xor_vec,
2409 .fno = gen_helper_gvec_xors,
2410 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2411 .vece = MO_64
2412 };
2413
2414 void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs,
2415 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2416 {
2417 TCGv_i64 tmp = tcg_temp_new_i64();
2418 gen_dup_i64(vece, tmp, c);
2419 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2420 tcg_temp_free_i64(tmp);
2421 }
2422
2423 void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs,
2424 int64_t c, uint32_t oprsz, uint32_t maxsz)
2425 {
2426 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2427 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2428 tcg_temp_free_i64(tmp);
2429 }
2430
2431 static const GVecGen2s gop_ors = {
2432 .fni8 = tcg_gen_or_i64,
2433 .fniv = tcg_gen_or_vec,
2434 .fno = gen_helper_gvec_ors,
2435 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2436 .vece = MO_64
2437 };
2438
2439 void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs,
2440 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2441 {
2442 TCGv_i64 tmp = tcg_temp_new_i64();
2443 gen_dup_i64(vece, tmp, c);
2444 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2445 tcg_temp_free_i64(tmp);
2446 }
2447
2448 void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs,
2449 int64_t c, uint32_t oprsz, uint32_t maxsz)
2450 {
2451 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2452 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2453 tcg_temp_free_i64(tmp);
2454 }
2455
2456 void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2457 {
2458 uint64_t mask = dup_const(MO_8, 0xff << c);
2459 tcg_gen_shli_i64(d, a, c);
2460 tcg_gen_andi_i64(d, d, mask);
2461 }
2462
2463 void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2464 {
2465 uint64_t mask = dup_const(MO_16, 0xffff << c);
2466 tcg_gen_shli_i64(d, a, c);
2467 tcg_gen_andi_i64(d, d, mask);
2468 }
2469
2470 void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs,
2471 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2472 {
2473 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
2474 static const GVecGen2i g[4] = {
2475 { .fni8 = tcg_gen_vec_shl8i_i64,
2476 .fniv = tcg_gen_shli_vec,
2477 .fno = gen_helper_gvec_shl8i,
2478 .opt_opc = vecop_list,
2479 .vece = MO_8 },
2480 { .fni8 = tcg_gen_vec_shl16i_i64,
2481 .fniv = tcg_gen_shli_vec,
2482 .fno = gen_helper_gvec_shl16i,
2483 .opt_opc = vecop_list,
2484 .vece = MO_16 },
2485 { .fni4 = tcg_gen_shli_i32,
2486 .fniv = tcg_gen_shli_vec,
2487 .fno = gen_helper_gvec_shl32i,
2488 .opt_opc = vecop_list,
2489 .vece = MO_32 },
2490 { .fni8 = tcg_gen_shli_i64,
2491 .fniv = tcg_gen_shli_vec,
2492 .fno = gen_helper_gvec_shl64i,
2493 .opt_opc = vecop_list,
2494 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2495 .vece = MO_64 },
2496 };
2497
2498 tcg_debug_assert(vece <= MO_64);
2499 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2500 if (shift == 0) {
2501 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2502 } else {
2503 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2504 }
2505 }
2506
2507 void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2508 {
2509 uint64_t mask = dup_const(MO_8, 0xff >> c);
2510 tcg_gen_shri_i64(d, a, c);
2511 tcg_gen_andi_i64(d, d, mask);
2512 }
2513
2514 void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2515 {
2516 uint64_t mask = dup_const(MO_16, 0xffff >> c);
2517 tcg_gen_shri_i64(d, a, c);
2518 tcg_gen_andi_i64(d, d, mask);
2519 }
2520
2521 void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs,
2522 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2523 {
2524 static const TCGOpcode vecop_list[] = { INDEX_op_shri_vec, 0 };
2525 static const GVecGen2i g[4] = {
2526 { .fni8 = tcg_gen_vec_shr8i_i64,
2527 .fniv = tcg_gen_shri_vec,
2528 .fno = gen_helper_gvec_shr8i,
2529 .opt_opc = vecop_list,
2530 .vece = MO_8 },
2531 { .fni8 = tcg_gen_vec_shr16i_i64,
2532 .fniv = tcg_gen_shri_vec,
2533 .fno = gen_helper_gvec_shr16i,
2534 .opt_opc = vecop_list,
2535 .vece = MO_16 },
2536 { .fni4 = tcg_gen_shri_i32,
2537 .fniv = tcg_gen_shri_vec,
2538 .fno = gen_helper_gvec_shr32i,
2539 .opt_opc = vecop_list,
2540 .vece = MO_32 },
2541 { .fni8 = tcg_gen_shri_i64,
2542 .fniv = tcg_gen_shri_vec,
2543 .fno = gen_helper_gvec_shr64i,
2544 .opt_opc = vecop_list,
2545 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2546 .vece = MO_64 },
2547 };
2548
2549 tcg_debug_assert(vece <= MO_64);
2550 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2551 if (shift == 0) {
2552 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2553 } else {
2554 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2555 }
2556 }
2557
2558 void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2559 {
2560 uint64_t s_mask = dup_const(MO_8, 0x80 >> c);
2561 uint64_t c_mask = dup_const(MO_8, 0xff >> c);
2562 TCGv_i64 s = tcg_temp_new_i64();
2563
2564 tcg_gen_shri_i64(d, a, c);
2565 tcg_gen_andi_i64(s, d, s_mask); /* isolate (shifted) sign bit */
2566 tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
2567 tcg_gen_andi_i64(d, d, c_mask); /* clear out bits above sign */
2568 tcg_gen_or_i64(d, d, s); /* include sign extension */
2569 tcg_temp_free_i64(s);
2570 }
2571
2572 void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2573 {
2574 uint64_t s_mask = dup_const(MO_16, 0x8000 >> c);
2575 uint64_t c_mask = dup_const(MO_16, 0xffff >> c);
2576 TCGv_i64 s = tcg_temp_new_i64();
2577
2578 tcg_gen_shri_i64(d, a, c);
2579 tcg_gen_andi_i64(s, d, s_mask); /* isolate (shifted) sign bit */
2580 tcg_gen_andi_i64(d, d, c_mask); /* clear out bits above sign */
2581 tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
2582 tcg_gen_or_i64(d, d, s); /* include sign extension */
2583 tcg_temp_free_i64(s);
2584 }
2585
2586 void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs,
2587 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2588 {
2589 static const TCGOpcode vecop_list[] = { INDEX_op_sari_vec, 0 };
2590 static const GVecGen2i g[4] = {
2591 { .fni8 = tcg_gen_vec_sar8i_i64,
2592 .fniv = tcg_gen_sari_vec,
2593 .fno = gen_helper_gvec_sar8i,
2594 .opt_opc = vecop_list,
2595 .vece = MO_8 },
2596 { .fni8 = tcg_gen_vec_sar16i_i64,
2597 .fniv = tcg_gen_sari_vec,
2598 .fno = gen_helper_gvec_sar16i,
2599 .opt_opc = vecop_list,
2600 .vece = MO_16 },
2601 { .fni4 = tcg_gen_sari_i32,
2602 .fniv = tcg_gen_sari_vec,
2603 .fno = gen_helper_gvec_sar32i,
2604 .opt_opc = vecop_list,
2605 .vece = MO_32 },
2606 { .fni8 = tcg_gen_sari_i64,
2607 .fniv = tcg_gen_sari_vec,
2608 .fno = gen_helper_gvec_sar64i,
2609 .opt_opc = vecop_list,
2610 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2611 .vece = MO_64 },
2612 };
2613
2614 tcg_debug_assert(vece <= MO_64);
2615 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2616 if (shift == 0) {
2617 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2618 } else {
2619 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2620 }
2621 }
2622
2623 /*
2624 * Specialized generation vector shifts by a non-constant scalar.
2625 */
2626
2627 typedef struct {
2628 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
2629 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
2630 void (*fniv_s)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32);
2631 void (*fniv_v)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
2632 gen_helper_gvec_2 *fno[4];
2633 TCGOpcode s_list[2];
2634 TCGOpcode v_list[2];
2635 } GVecGen2sh;
2636
2637 static void expand_2sh_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
2638 uint32_t oprsz, uint32_t tysz, TCGType type,
2639 TCGv_i32 shift,
2640 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32))
2641 {
2642 TCGv_vec t0 = tcg_temp_new_vec(type);
2643 uint32_t i;
2644
2645 for (i = 0; i < oprsz; i += tysz) {
2646 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
2647 fni(vece, t0, t0, shift);
2648 tcg_gen_st_vec(t0, cpu_env, dofs + i);
2649 }
2650 tcg_temp_free_vec(t0);
2651 }
2652
2653 static void
2654 do_gvec_shifts(unsigned vece, uint32_t dofs, uint32_t aofs, TCGv_i32 shift,
2655 uint32_t oprsz, uint32_t maxsz, const GVecGen2sh *g)
2656 {
2657 TCGType type;
2658 uint32_t some;
2659
2660 check_size_align(oprsz, maxsz, dofs | aofs);
2661 check_overlap_2(dofs, aofs, maxsz);
2662
2663 /* If the backend has a scalar expansion, great. */
2664 type = choose_vector_type(g->s_list, vece, oprsz, vece == MO_64);
2665 if (type) {
2666 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
2667 switch (type) {
2668 case TCG_TYPE_V256:
2669 some = QEMU_ALIGN_DOWN(oprsz, 32);
2670 expand_2sh_vec(vece, dofs, aofs, some, 32,
2671 TCG_TYPE_V256, shift, g->fniv_s);
2672 if (some == oprsz) {
2673 break;
2674 }
2675 dofs += some;
2676 aofs += some;
2677 oprsz -= some;
2678 maxsz -= some;
2679 /* fallthru */
2680 case TCG_TYPE_V128:
2681 expand_2sh_vec(vece, dofs, aofs, oprsz, 16,
2682 TCG_TYPE_V128, shift, g->fniv_s);
2683 break;
2684 case TCG_TYPE_V64:
2685 expand_2sh_vec(vece, dofs, aofs, oprsz, 8,
2686 TCG_TYPE_V64, shift, g->fniv_s);
2687 break;
2688 default:
2689 g_assert_not_reached();
2690 }
2691 tcg_swap_vecop_list(hold_list);
2692 goto clear_tail;
2693 }
2694
2695 /* If the backend supports variable vector shifts, also cool. */
2696 type = choose_vector_type(g->v_list, vece, oprsz, vece == MO_64);
2697 if (type) {
2698 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
2699 TCGv_vec v_shift = tcg_temp_new_vec(type);
2700
2701 if (vece == MO_64) {
2702 TCGv_i64 sh64 = tcg_temp_new_i64();
2703 tcg_gen_extu_i32_i64(sh64, shift);
2704 tcg_gen_dup_i64_vec(MO_64, v_shift, sh64);
2705 tcg_temp_free_i64(sh64);
2706 } else {
2707 tcg_gen_dup_i32_vec(vece, v_shift, shift);
2708 }
2709
2710 switch (type) {
2711 case TCG_TYPE_V256:
2712 some = QEMU_ALIGN_DOWN(oprsz, 32);
2713 expand_2s_vec(vece, dofs, aofs, some, 32, TCG_TYPE_V256,
2714 v_shift, false, g->fniv_v);
2715 if (some == oprsz) {
2716 break;
2717 }
2718 dofs += some;
2719 aofs += some;
2720 oprsz -= some;
2721 maxsz -= some;
2722 /* fallthru */
2723 case TCG_TYPE_V128:
2724 expand_2s_vec(vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
2725 v_shift, false, g->fniv_v);
2726 break;
2727 case TCG_TYPE_V64:
2728 expand_2s_vec(vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
2729 v_shift, false, g->fniv_v);
2730 break;
2731 default:
2732 g_assert_not_reached();
2733 }
2734 tcg_temp_free_vec(v_shift);
2735 tcg_swap_vecop_list(hold_list);
2736 goto clear_tail;
2737 }
2738
2739 /* Otherwise fall back to integral... */
2740 if (vece == MO_32 && check_size_impl(oprsz, 4)) {
2741 expand_2s_i32(dofs, aofs, oprsz, shift, false, g->fni4);
2742 } else if (vece == MO_64 && check_size_impl(oprsz, 8)) {
2743 TCGv_i64 sh64 = tcg_temp_new_i64();
2744 tcg_gen_extu_i32_i64(sh64, shift);
2745 expand_2s_i64(dofs, aofs, oprsz, sh64, false, g->fni8);
2746 tcg_temp_free_i64(sh64);
2747 } else {
2748 TCGv_ptr a0 = tcg_temp_new_ptr();
2749 TCGv_ptr a1 = tcg_temp_new_ptr();
2750 TCGv_i32 desc = tcg_temp_new_i32();
2751
2752 tcg_gen_shli_i32(desc, shift, SIMD_DATA_SHIFT);
2753 tcg_gen_ori_i32(desc, desc, simd_desc(oprsz, maxsz, 0));
2754 tcg_gen_addi_ptr(a0, cpu_env, dofs);
2755 tcg_gen_addi_ptr(a1, cpu_env, aofs);
2756
2757 g->fno[vece](a0, a1, desc);
2758
2759 tcg_temp_free_ptr(a0);
2760 tcg_temp_free_ptr(a1);
2761 tcg_temp_free_i32(desc);
2762 return;
2763 }
2764
2765 clear_tail:
2766 if (oprsz < maxsz) {
2767 expand_clr(dofs + oprsz, maxsz - oprsz);
2768 }
2769 }
2770
2771 void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs,
2772 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2773 {
2774 static const GVecGen2sh g = {
2775 .fni4 = tcg_gen_shl_i32,
2776 .fni8 = tcg_gen_shl_i64,
2777 .fniv_s = tcg_gen_shls_vec,
2778 .fniv_v = tcg_gen_shlv_vec,
2779 .fno = {
2780 gen_helper_gvec_shl8i,
2781 gen_helper_gvec_shl16i,
2782 gen_helper_gvec_shl32i,
2783 gen_helper_gvec_shl64i,
2784 },
2785 .s_list = { INDEX_op_shls_vec, 0 },
2786 .v_list = { INDEX_op_shlv_vec, 0 },
2787 };
2788
2789 tcg_debug_assert(vece <= MO_64);
2790 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2791 }
2792
2793 void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs,
2794 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2795 {
2796 static const GVecGen2sh g = {
2797 .fni4 = tcg_gen_shr_i32,
2798 .fni8 = tcg_gen_shr_i64,
2799 .fniv_s = tcg_gen_shrs_vec,
2800 .fniv_v = tcg_gen_shrv_vec,
2801 .fno = {
2802 gen_helper_gvec_shr8i,
2803 gen_helper_gvec_shr16i,
2804 gen_helper_gvec_shr32i,
2805 gen_helper_gvec_shr64i,
2806 },
2807 .s_list = { INDEX_op_shrs_vec, 0 },
2808 .v_list = { INDEX_op_shrv_vec, 0 },
2809 };
2810
2811 tcg_debug_assert(vece <= MO_64);
2812 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2813 }
2814
2815 void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs,
2816 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2817 {
2818 static const GVecGen2sh g = {
2819 .fni4 = tcg_gen_sar_i32,
2820 .fni8 = tcg_gen_sar_i64,
2821 .fniv_s = tcg_gen_sars_vec,
2822 .fniv_v = tcg_gen_sarv_vec,
2823 .fno = {
2824 gen_helper_gvec_sar8i,
2825 gen_helper_gvec_sar16i,
2826 gen_helper_gvec_sar32i,
2827 gen_helper_gvec_sar64i,
2828 },
2829 .s_list = { INDEX_op_sars_vec, 0 },
2830 .v_list = { INDEX_op_sarv_vec, 0 },
2831 };
2832
2833 tcg_debug_assert(vece <= MO_64);
2834 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2835 }
2836
2837 /*
2838 * Expand D = A << (B % element bits)
2839 *
2840 * Unlike scalar shifts, where it is easy for the target front end
2841 * to include the modulo as part of the expansion. If the target
2842 * naturally includes the modulo as part of the operation, great!
2843 * If the target has some other behaviour from out-of-range shifts,
2844 * then it could not use this function anyway, and would need to
2845 * do it's own expansion with custom functions.
2846 */
2847 static void tcg_gen_shlv_mod_vec(unsigned vece, TCGv_vec d,
2848 TCGv_vec a, TCGv_vec b)
2849 {
2850 TCGv_vec t = tcg_temp_new_vec_matching(d);
2851
2852 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2853 tcg_gen_and_vec(vece, t, t, b);
2854 tcg_gen_shlv_vec(vece, d, a, t);
2855 tcg_temp_free_vec(t);
2856 }
2857
2858 static void tcg_gen_shl_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2859 {
2860 TCGv_i32 t = tcg_temp_new_i32();
2861
2862 tcg_gen_andi_i32(t, b, 31);
2863 tcg_gen_shl_i32(d, a, t);
2864 tcg_temp_free_i32(t);
2865 }
2866
2867 static void tcg_gen_shl_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2868 {
2869 TCGv_i64 t = tcg_temp_new_i64();
2870
2871 tcg_gen_andi_i64(t, b, 63);
2872 tcg_gen_shl_i64(d, a, t);
2873 tcg_temp_free_i64(t);
2874 }
2875
2876 void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs,
2877 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2878 {
2879 static const TCGOpcode vecop_list[] = { INDEX_op_shlv_vec, 0 };
2880 static const GVecGen3 g[4] = {
2881 { .fniv = tcg_gen_shlv_mod_vec,
2882 .fno = gen_helper_gvec_shl8v,
2883 .opt_opc = vecop_list,
2884 .vece = MO_8 },
2885 { .fniv = tcg_gen_shlv_mod_vec,
2886 .fno = gen_helper_gvec_shl16v,
2887 .opt_opc = vecop_list,
2888 .vece = MO_16 },
2889 { .fni4 = tcg_gen_shl_mod_i32,
2890 .fniv = tcg_gen_shlv_mod_vec,
2891 .fno = gen_helper_gvec_shl32v,
2892 .opt_opc = vecop_list,
2893 .vece = MO_32 },
2894 { .fni8 = tcg_gen_shl_mod_i64,
2895 .fniv = tcg_gen_shlv_mod_vec,
2896 .fno = gen_helper_gvec_shl64v,
2897 .opt_opc = vecop_list,
2898 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2899 .vece = MO_64 },
2900 };
2901
2902 tcg_debug_assert(vece <= MO_64);
2903 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2904 }
2905
2906 /*
2907 * Similarly for logical right shifts.
2908 */
2909
2910 static void tcg_gen_shrv_mod_vec(unsigned vece, TCGv_vec d,
2911 TCGv_vec a, TCGv_vec b)
2912 {
2913 TCGv_vec t = tcg_temp_new_vec_matching(d);
2914
2915 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2916 tcg_gen_and_vec(vece, t, t, b);
2917 tcg_gen_shrv_vec(vece, d, a, t);
2918 tcg_temp_free_vec(t);
2919 }
2920
2921 static void tcg_gen_shr_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2922 {
2923 TCGv_i32 t = tcg_temp_new_i32();
2924
2925 tcg_gen_andi_i32(t, b, 31);
2926 tcg_gen_shr_i32(d, a, t);
2927 tcg_temp_free_i32(t);
2928 }
2929
2930 static void tcg_gen_shr_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2931 {
2932 TCGv_i64 t = tcg_temp_new_i64();
2933
2934 tcg_gen_andi_i64(t, b, 63);
2935 tcg_gen_shr_i64(d, a, t);
2936 tcg_temp_free_i64(t);
2937 }
2938
2939 void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs,
2940 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2941 {
2942 static const TCGOpcode vecop_list[] = { INDEX_op_shrv_vec, 0 };
2943 static const GVecGen3 g[4] = {
2944 { .fniv = tcg_gen_shrv_mod_vec,
2945 .fno = gen_helper_gvec_shr8v,
2946 .opt_opc = vecop_list,
2947 .vece = MO_8 },
2948 { .fniv = tcg_gen_shrv_mod_vec,
2949 .fno = gen_helper_gvec_shr16v,
2950 .opt_opc = vecop_list,
2951 .vece = MO_16 },
2952 { .fni4 = tcg_gen_shr_mod_i32,
2953 .fniv = tcg_gen_shrv_mod_vec,
2954 .fno = gen_helper_gvec_shr32v,
2955 .opt_opc = vecop_list,
2956 .vece = MO_32 },
2957 { .fni8 = tcg_gen_shr_mod_i64,
2958 .fniv = tcg_gen_shrv_mod_vec,
2959 .fno = gen_helper_gvec_shr64v,
2960 .opt_opc = vecop_list,
2961 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2962 .vece = MO_64 },
2963 };
2964
2965 tcg_debug_assert(vece <= MO_64);
2966 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2967 }
2968
2969 /*
2970 * Similarly for arithmetic right shifts.
2971 */
2972
2973 static void tcg_gen_sarv_mod_vec(unsigned vece, TCGv_vec d,
2974 TCGv_vec a, TCGv_vec b)
2975 {
2976 TCGv_vec t = tcg_temp_new_vec_matching(d);
2977
2978 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2979 tcg_gen_and_vec(vece, t, t, b);
2980 tcg_gen_sarv_vec(vece, d, a, t);
2981 tcg_temp_free_vec(t);
2982 }
2983
2984 static void tcg_gen_sar_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2985 {
2986 TCGv_i32 t = tcg_temp_new_i32();
2987
2988 tcg_gen_andi_i32(t, b, 31);
2989 tcg_gen_sar_i32(d, a, t);
2990 tcg_temp_free_i32(t);
2991 }
2992
2993 static void tcg_gen_sar_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2994 {
2995 TCGv_i64 t = tcg_temp_new_i64();
2996
2997 tcg_gen_andi_i64(t, b, 63);
2998 tcg_gen_sar_i64(d, a, t);
2999 tcg_temp_free_i64(t);
3000 }
3001
3002 void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs,
3003 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3004 {
3005 static const TCGOpcode vecop_list[] = { INDEX_op_sarv_vec, 0 };
3006 static const GVecGen3 g[4] = {
3007 { .fniv = tcg_gen_sarv_mod_vec,
3008 .fno = gen_helper_gvec_sar8v,
3009 .opt_opc = vecop_list,
3010 .vece = MO_8 },
3011 { .fniv = tcg_gen_sarv_mod_vec,
3012 .fno = gen_helper_gvec_sar16v,
3013 .opt_opc = vecop_list,
3014 .vece = MO_16 },
3015 { .fni4 = tcg_gen_sar_mod_i32,
3016 .fniv = tcg_gen_sarv_mod_vec,
3017 .fno = gen_helper_gvec_sar32v,
3018 .opt_opc = vecop_list,
3019 .vece = MO_32 },
3020 { .fni8 = tcg_gen_sar_mod_i64,
3021 .fniv = tcg_gen_sarv_mod_vec,
3022 .fno = gen_helper_gvec_sar64v,
3023 .opt_opc = vecop_list,
3024 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3025 .vece = MO_64 },
3026 };
3027
3028 tcg_debug_assert(vece <= MO_64);
3029 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3030 }
3031
3032 /* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
3033 static void expand_cmp_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3034 uint32_t oprsz, TCGCond cond)
3035 {
3036 TCGv_i32 t0 = tcg_temp_new_i32();
3037 TCGv_i32 t1 = tcg_temp_new_i32();
3038 uint32_t i;
3039
3040 for (i = 0; i < oprsz; i += 4) {
3041 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
3042 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
3043 tcg_gen_setcond_i32(cond, t0, t0, t1);
3044 tcg_gen_neg_i32(t0, t0);
3045 tcg_gen_st_i32(t0, cpu_env, dofs + i);
3046 }
3047 tcg_temp_free_i32(t1);
3048 tcg_temp_free_i32(t0);
3049 }
3050
3051 static void expand_cmp_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3052 uint32_t oprsz, TCGCond cond)
3053 {
3054 TCGv_i64 t0 = tcg_temp_new_i64();
3055 TCGv_i64 t1 = tcg_temp_new_i64();
3056 uint32_t i;
3057
3058 for (i = 0; i < oprsz; i += 8) {
3059 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
3060 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
3061 tcg_gen_setcond_i64(cond, t0, t0, t1);
3062 tcg_gen_neg_i64(t0, t0);
3063 tcg_gen_st_i64(t0, cpu_env, dofs + i);
3064 }
3065 tcg_temp_free_i64(t1);
3066 tcg_temp_free_i64(t0);
3067 }
3068
3069 static void expand_cmp_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
3070 uint32_t bofs, uint32_t oprsz, uint32_t tysz,
3071 TCGType type, TCGCond cond)
3072 {
3073 TCGv_vec t0 = tcg_temp_new_vec(type);
3074 TCGv_vec t1 = tcg_temp_new_vec(type);
3075 uint32_t i;
3076
3077 for (i = 0; i < oprsz; i += tysz) {
3078 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
3079 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
3080 tcg_gen_cmp_vec(cond, vece, t0, t0, t1);
3081 tcg_gen_st_vec(t0, cpu_env, dofs + i);
3082 }
3083 tcg_temp_free_vec(t1);
3084 tcg_temp_free_vec(t0);
3085 }
3086
3087 void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs,
3088 uint32_t aofs, uint32_t bofs,
3089 uint32_t oprsz, uint32_t maxsz)
3090 {
3091 static const TCGOpcode cmp_list[] = { INDEX_op_cmp_vec, 0 };
3092 static gen_helper_gvec_3 * const eq_fn[4] = {
3093 gen_helper_gvec_eq8, gen_helper_gvec_eq16,
3094 gen_helper_gvec_eq32, gen_helper_gvec_eq64
3095 };
3096 static gen_helper_gvec_3 * const ne_fn[4] = {
3097 gen_helper_gvec_ne8, gen_helper_gvec_ne16,
3098 gen_helper_gvec_ne32, gen_helper_gvec_ne64
3099 };
3100 static gen_helper_gvec_3 * const lt_fn[4] = {
3101 gen_helper_gvec_lt8, gen_helper_gvec_lt16,
3102 gen_helper_gvec_lt32, gen_helper_gvec_lt64
3103 };
3104 static gen_helper_gvec_3 * const le_fn[4] = {
3105 gen_helper_gvec_le8, gen_helper_gvec_le16,
3106 gen_helper_gvec_le32, gen_helper_gvec_le64
3107 };
3108 static gen_helper_gvec_3 * const ltu_fn[4] = {
3109 gen_helper_gvec_ltu8, gen_helper_gvec_ltu16,
3110 gen_helper_gvec_ltu32, gen_helper_gvec_ltu64
3111 };
3112 static gen_helper_gvec_3 * const leu_fn[4] = {
3113 gen_helper_gvec_leu8, gen_helper_gvec_leu16,
3114 gen_helper_gvec_leu32, gen_helper_gvec_leu64
3115 };
3116 static gen_helper_gvec_3 * const * const fns[16] = {
3117 [TCG_COND_EQ] = eq_fn,
3118 [TCG_COND_NE] = ne_fn,
3119 [TCG_COND_LT] = lt_fn,
3120 [TCG_COND_LE] = le_fn,
3121 [TCG_COND_LTU] = ltu_fn,
3122 [TCG_COND_LEU] = leu_fn,
3123 };
3124
3125 const TCGOpcode *hold_list;
3126 TCGType type;
3127 uint32_t some;
3128
3129 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
3130 check_overlap_3(dofs, aofs, bofs, maxsz);
3131
3132 if (cond == TCG_COND_NEVER || cond == TCG_COND_ALWAYS) {
3133 do_dup(MO_8, dofs, oprsz, maxsz,
3134 NULL, NULL, -(cond == TCG_COND_ALWAYS));
3135 return;
3136 }
3137
3138 /*
3139 * Implement inline with a vector type, if possible.
3140 * Prefer integer when 64-bit host and 64-bit comparison.
3141 */
3142 hold_list = tcg_swap_vecop_list(cmp_list);
3143 type = choose_vector_type(cmp_list, vece, oprsz,
3144 TCG_TARGET_REG_BITS == 64 && vece == MO_64);
3145 switch (type) {
3146 case TCG_TYPE_V256:
3147 /* Recall that ARM SVE allows vector sizes that are not a
3148 * power of 2, but always a multiple of 16. The intent is
3149 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
3150 */
3151 some = QEMU_ALIGN_DOWN(oprsz, 32);
3152 expand_cmp_vec(vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256, cond);
3153 if (some == oprsz) {
3154 break;
3155 }
3156 dofs += some;
3157 aofs += some;
3158 bofs += some;
3159 oprsz -= some;
3160 maxsz -= some;
3161 /* fallthru */
3162 case TCG_TYPE_V128:
3163 expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128, cond);
3164 break;
3165 case TCG_TYPE_V64:
3166 expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64, cond);
3167 break;
3168
3169 case 0:
3170 if (vece == MO_64 && check_size_impl(oprsz, 8)) {
3171 expand_cmp_i64(dofs, aofs, bofs, oprsz, cond);
3172 } else if (vece == MO_32 && check_size_impl(oprsz, 4)) {
3173 expand_cmp_i32(dofs, aofs, bofs, oprsz, cond);
3174 } else {
3175 gen_helper_gvec_3 * const *fn = fns[cond];
3176
3177 if (fn == NULL) {
3178 uint32_t tmp;
3179 tmp = aofs, aofs = bofs, bofs = tmp;
3180 cond = tcg_swap_cond(cond);
3181 fn = fns[cond];
3182 assert(fn != NULL);
3183 }
3184 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, 0, fn[vece]);
3185 oprsz = maxsz;
3186 }
3187 break;
3188
3189 default:
3190 g_assert_not_reached();
3191 }
3192 tcg_swap_vecop_list(hold_list);
3193
3194 if (oprsz < maxsz) {
3195 expand_clr(dofs + oprsz, maxsz - oprsz);
3196 }
3197 }
3198
3199 static void tcg_gen_bitsel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c)
3200 {
3201 TCGv_i64 t = tcg_temp_new_i64();
3202
3203 tcg_gen_and_i64(t, b, a);
3204 tcg_gen_andc_i64(d, c, a);
3205 tcg_gen_or_i64(d, d, t);
3206 tcg_temp_free_i64(t);
3207 }
3208
3209 void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs,
3210 uint32_t bofs, uint32_t cofs,
3211 uint32_t oprsz, uint32_t maxsz)
3212 {
3213 static const GVecGen4 g = {
3214 .fni8 = tcg_gen_bitsel_i64,
3215 .fniv = tcg_gen_bitsel_vec,
3216 .fno = gen_helper_gvec_bitsel,
3217 };
3218
3219 tcg_gen_gvec_4(dofs, aofs, bofs, cofs, oprsz, maxsz, &g);
3220 }
armbru's QEMU hackery