?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
s390x/tcg: Implement VECTOR FP COMPARE (EQUAL|HIGH|HIGH OR EQUAL)
[qemu/ar7.git] / target / s390x / translate_vx.inc.c
blob5571a71e1a480cc821f52621791e0c5e85d7b898
1 /*
2 * QEMU TCG support -- s390x vector instruction translation functions
3 *
4 * Copyright (C) 2019 Red Hat Inc
5 *
6 * Authors:
7 * David Hildenbrand <david@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13 /*
14 * For most instructions that use the same element size for reads and
15 * writes, we can use real gvec vector expansion, which potantially uses
16 * real host vector instructions. As they only work up to 64 bit elements,
17 * 128 bit elements (vector is a single element) have to be handled
18 * differently. Operations that are too complicated to encode via TCG ops
19 * are handled via gvec ool (out-of-line) handlers.
20 *
21 * As soon as instructions use different element sizes for reads and writes
22 * or access elements "out of their element scope" we expand them manually
23 * in fancy loops, as gvec expansion does not deal with actual element
24 * numbers and does also not support access to other elements.
25 *
26 * 128 bit elements:
27 * As we only have i32/i64, such elements have to be loaded into two
28 * i64 values and can then be processed e.g. by tcg_gen_add2_i64.
29 *
30 * Sizes:
31 * On s390x, the operand size (oprsz) and the maximum size (maxsz) are
32 * always 16 (128 bit). What gvec code calls "vece", s390x calls "es",
33 * a.k.a. "element size". These values nicely map to MO_8 ... MO_64. Only
34 * 128 bit element size has to be treated in a special way (MO_64 + 1).
35 * We will use ES_* instead of MO_* for this reason in this file.
36 *
37 * CC handling:
38 * As gvec ool-helpers can currently not return values (besides via
39 * pointers like vectors or cpu_env), whenever we have to set the CC and
40 * can't conclude the value from the result vector, we will directly
41 * set it in "env->cc_op" and mark it as static via set_cc_static()".
42 * Whenever this is done, the helper writes globals (cc_op).
43 */
44
45 #define NUM_VEC_ELEMENT_BYTES(es) (1 << (es))
46 #define NUM_VEC_ELEMENTS(es) (16 / NUM_VEC_ELEMENT_BYTES(es))
47 #define NUM_VEC_ELEMENT_BITS(es) (NUM_VEC_ELEMENT_BYTES(es) * BITS_PER_BYTE)
48
49 #define ES_8 MO_8
50 #define ES_16 MO_16
51 #define ES_32 MO_32
52 #define ES_64 MO_64
53 #define ES_128 4
54
55 /* Floating-Point Format */
56 #define FPF_SHORT 2
57 #define FPF_LONG 3
58 #define FPF_EXT 4
59
60 static inline bool valid_vec_element(uint8_t enr, TCGMemOp es)
61 {
62 return !(enr & ~(NUM_VEC_ELEMENTS(es) - 1));
63 }
64
65 static void read_vec_element_i64(TCGv_i64 dst, uint8_t reg, uint8_t enr,
66 TCGMemOp memop)
67 {
68 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
69
70 switch (memop) {
71 case ES_8:
72 tcg_gen_ld8u_i64(dst, cpu_env, offs);
73 break;
74 case ES_16:
75 tcg_gen_ld16u_i64(dst, cpu_env, offs);
76 break;
77 case ES_32:
78 tcg_gen_ld32u_i64(dst, cpu_env, offs);
79 break;
80 case ES_8 | MO_SIGN:
81 tcg_gen_ld8s_i64(dst, cpu_env, offs);
82 break;
83 case ES_16 | MO_SIGN:
84 tcg_gen_ld16s_i64(dst, cpu_env, offs);
85 break;
86 case ES_32 | MO_SIGN:
87 tcg_gen_ld32s_i64(dst, cpu_env, offs);
88 break;
89 case ES_64:
90 case ES_64 | MO_SIGN:
91 tcg_gen_ld_i64(dst, cpu_env, offs);
92 break;
93 default:
94 g_assert_not_reached();
95 }
96 }
97
98 static void read_vec_element_i32(TCGv_i32 dst, uint8_t reg, uint8_t enr,
99 TCGMemOp memop)
100 {
101 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
102
103 switch (memop) {
104 case ES_8:
105 tcg_gen_ld8u_i32(dst, cpu_env, offs);
106 break;
107 case ES_16:
108 tcg_gen_ld16u_i32(dst, cpu_env, offs);
109 break;
110 case ES_8 | MO_SIGN:
111 tcg_gen_ld8s_i32(dst, cpu_env, offs);
112 break;
113 case ES_16 | MO_SIGN:
114 tcg_gen_ld16s_i32(dst, cpu_env, offs);
115 break;
116 case ES_32:
117 case ES_32 | MO_SIGN:
118 tcg_gen_ld_i32(dst, cpu_env, offs);
119 break;
120 default:
121 g_assert_not_reached();
122 }
123 }
124
125 static void write_vec_element_i64(TCGv_i64 src, int reg, uint8_t enr,
126 TCGMemOp memop)
127 {
128 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
129
130 switch (memop) {
131 case ES_8:
132 tcg_gen_st8_i64(src, cpu_env, offs);
133 break;
134 case ES_16:
135 tcg_gen_st16_i64(src, cpu_env, offs);
136 break;
137 case ES_32:
138 tcg_gen_st32_i64(src, cpu_env, offs);
139 break;
140 case ES_64:
141 tcg_gen_st_i64(src, cpu_env, offs);
142 break;
143 default:
144 g_assert_not_reached();
145 }
146 }
147
148 static void write_vec_element_i32(TCGv_i32 src, int reg, uint8_t enr,
149 TCGMemOp memop)
150 {
151 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
152
153 switch (memop) {
154 case ES_8:
155 tcg_gen_st8_i32(src, cpu_env, offs);
156 break;
157 case ES_16:
158 tcg_gen_st16_i32(src, cpu_env, offs);
159 break;
160 case ES_32:
161 tcg_gen_st_i32(src, cpu_env, offs);
162 break;
163 default:
164 g_assert_not_reached();
165 }
166 }
167
168 static void get_vec_element_ptr_i64(TCGv_ptr ptr, uint8_t reg, TCGv_i64 enr,
169 uint8_t es)
170 {
171 TCGv_i64 tmp = tcg_temp_new_i64();
172
173 /* mask off invalid parts from the element nr */
174 tcg_gen_andi_i64(tmp, enr, NUM_VEC_ELEMENTS(es) - 1);
175
176 /* convert it to an element offset relative to cpu_env (vec_reg_offset() */
177 tcg_gen_shli_i64(tmp, tmp, es);
178 #ifndef HOST_WORDS_BIGENDIAN
179 tcg_gen_xori_i64(tmp, tmp, 8 - NUM_VEC_ELEMENT_BYTES(es));
180 #endif
181 tcg_gen_addi_i64(tmp, tmp, vec_full_reg_offset(reg));
182
183 /* generate the final ptr by adding cpu_env */
184 tcg_gen_trunc_i64_ptr(ptr, tmp);
185 tcg_gen_add_ptr(ptr, ptr, cpu_env);
186
187 tcg_temp_free_i64(tmp);
188 }
189
190 #define gen_gvec_2(v1, v2, gen) \
191 tcg_gen_gvec_2(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
192 16, 16, gen)
193 #define gen_gvec_2s(v1, v2, c, gen) \
194 tcg_gen_gvec_2s(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
195 16, 16, c, gen)
196 #define gen_gvec_2_ool(v1, v2, data, fn) \
197 tcg_gen_gvec_2_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
198 16, 16, data, fn)
199 #define gen_gvec_2i_ool(v1, v2, c, data, fn) \
200 tcg_gen_gvec_2i_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
201 c, 16, 16, data, fn)
202 #define gen_gvec_2_ptr(v1, v2, ptr, data, fn) \
203 tcg_gen_gvec_2_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
204 ptr, 16, 16, data, fn)
205 #define gen_gvec_3(v1, v2, v3, gen) \
206 tcg_gen_gvec_3(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
207 vec_full_reg_offset(v3), 16, 16, gen)
208 #define gen_gvec_3_ool(v1, v2, v3, data, fn) \
209 tcg_gen_gvec_3_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
210 vec_full_reg_offset(v3), 16, 16, data, fn)
211 #define gen_gvec_3_ptr(v1, v2, v3, ptr, data, fn) \
212 tcg_gen_gvec_3_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
213 vec_full_reg_offset(v3), ptr, 16, 16, data, fn)
214 #define gen_gvec_3i(v1, v2, v3, c, gen) \
215 tcg_gen_gvec_3i(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
216 vec_full_reg_offset(v3), c, 16, 16, gen)
217 #define gen_gvec_4(v1, v2, v3, v4, gen) \
218 tcg_gen_gvec_4(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
219 vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
220 16, 16, gen)
221 #define gen_gvec_4_ool(v1, v2, v3, v4, data, fn) \
222 tcg_gen_gvec_4_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
223 vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
224 16, 16, data, fn)
225 #define gen_gvec_4_ptr(v1, v2, v3, v4, ptr, data, fn) \
226 tcg_gen_gvec_4_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
227 vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
228 ptr, 16, 16, data, fn)
229 #define gen_gvec_dup_i64(es, v1, c) \
230 tcg_gen_gvec_dup_i64(es, vec_full_reg_offset(v1), 16, 16, c)
231 #define gen_gvec_mov(v1, v2) \
232 tcg_gen_gvec_mov(0, vec_full_reg_offset(v1), vec_full_reg_offset(v2), 16, \
233 16)
234 #define gen_gvec_dup64i(v1, c) \
235 tcg_gen_gvec_dup64i(vec_full_reg_offset(v1), 16, 16, c)
236 #define gen_gvec_fn_2(fn, es, v1, v2) \
237 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
238 16, 16)
239 #define gen_gvec_fn_2i(fn, es, v1, v2, c) \
240 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
241 c, 16, 16)
242 #define gen_gvec_fn_2s(fn, es, v1, v2, s) \
243 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
244 s, 16, 16)
245 #define gen_gvec_fn_3(fn, es, v1, v2, v3) \
246 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
247 vec_full_reg_offset(v3), 16, 16)
248
249 /*
250 * Helper to carry out a 128 bit vector computation using 2 i64 values per
251 * vector.
252 */
253 typedef void (*gen_gvec128_3_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
254 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh);
255 static void gen_gvec128_3_i64(gen_gvec128_3_i64_fn fn, uint8_t d, uint8_t a,
256 uint8_t b)
257 {
258 TCGv_i64 dh = tcg_temp_new_i64();
259 TCGv_i64 dl = tcg_temp_new_i64();
260 TCGv_i64 ah = tcg_temp_new_i64();
261 TCGv_i64 al = tcg_temp_new_i64();
262 TCGv_i64 bh = tcg_temp_new_i64();
263 TCGv_i64 bl = tcg_temp_new_i64();
264
265 read_vec_element_i64(ah, a, 0, ES_64);
266 read_vec_element_i64(al, a, 1, ES_64);
267 read_vec_element_i64(bh, b, 0, ES_64);
268 read_vec_element_i64(bl, b, 1, ES_64);
269 fn(dl, dh, al, ah, bl, bh);
270 write_vec_element_i64(dh, d, 0, ES_64);
271 write_vec_element_i64(dl, d, 1, ES_64);
272
273 tcg_temp_free_i64(dh);
274 tcg_temp_free_i64(dl);
275 tcg_temp_free_i64(ah);
276 tcg_temp_free_i64(al);
277 tcg_temp_free_i64(bh);
278 tcg_temp_free_i64(bl);
279 }
280
281 typedef void (*gen_gvec128_4_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
282 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh,
283 TCGv_i64 cl, TCGv_i64 ch);
284 static void gen_gvec128_4_i64(gen_gvec128_4_i64_fn fn, uint8_t d, uint8_t a,
285 uint8_t b, uint8_t c)
286 {
287 TCGv_i64 dh = tcg_temp_new_i64();
288 TCGv_i64 dl = tcg_temp_new_i64();
289 TCGv_i64 ah = tcg_temp_new_i64();
290 TCGv_i64 al = tcg_temp_new_i64();
291 TCGv_i64 bh = tcg_temp_new_i64();
292 TCGv_i64 bl = tcg_temp_new_i64();
293 TCGv_i64 ch = tcg_temp_new_i64();
294 TCGv_i64 cl = tcg_temp_new_i64();
295
296 read_vec_element_i64(ah, a, 0, ES_64);
297 read_vec_element_i64(al, a, 1, ES_64);
298 read_vec_element_i64(bh, b, 0, ES_64);
299 read_vec_element_i64(bl, b, 1, ES_64);
300 read_vec_element_i64(ch, c, 0, ES_64);
301 read_vec_element_i64(cl, c, 1, ES_64);
302 fn(dl, dh, al, ah, bl, bh, cl, ch);
303 write_vec_element_i64(dh, d, 0, ES_64);
304 write_vec_element_i64(dl, d, 1, ES_64);
305
306 tcg_temp_free_i64(dh);
307 tcg_temp_free_i64(dl);
308 tcg_temp_free_i64(ah);
309 tcg_temp_free_i64(al);
310 tcg_temp_free_i64(bh);
311 tcg_temp_free_i64(bl);
312 tcg_temp_free_i64(ch);
313 tcg_temp_free_i64(cl);
314 }
315
316 static void gen_gvec_dupi(uint8_t es, uint8_t reg, uint64_t c)
317 {
318 switch (es) {
319 case ES_8:
320 tcg_gen_gvec_dup8i(vec_full_reg_offset(reg), 16, 16, c);
321 break;
322 case ES_16:
323 tcg_gen_gvec_dup16i(vec_full_reg_offset(reg), 16, 16, c);
324 break;
325 case ES_32:
326 tcg_gen_gvec_dup32i(vec_full_reg_offset(reg), 16, 16, c);
327 break;
328 case ES_64:
329 gen_gvec_dup64i(reg, c);
330 break;
331 default:
332 g_assert_not_reached();
333 }
334 }
335
336 static void zero_vec(uint8_t reg)
337 {
338 tcg_gen_gvec_dup8i(vec_full_reg_offset(reg), 16, 16, 0);
339 }
340
341 static void gen_addi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
342 uint64_t b)
343 {
344 TCGv_i64 bl = tcg_const_i64(b);
345 TCGv_i64 bh = tcg_const_i64(0);
346
347 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
348 tcg_temp_free_i64(bl);
349 tcg_temp_free_i64(bh);
350 }
351
352 static DisasJumpType op_vge(DisasContext *s, DisasOps *o)
353 {
354 const uint8_t es = s->insn->data;
355 const uint8_t enr = get_field(s->fields, m3);
356 TCGv_i64 tmp;
357
358 if (!valid_vec_element(enr, es)) {
359 gen_program_exception(s, PGM_SPECIFICATION);
360 return DISAS_NORETURN;
361 }
362
363 tmp = tcg_temp_new_i64();
364 read_vec_element_i64(tmp, get_field(s->fields, v2), enr, es);
365 tcg_gen_add_i64(o->addr1, o->addr1, tmp);
366 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
367
368 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
369 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
370 tcg_temp_free_i64(tmp);
371 return DISAS_NEXT;
372 }
373
374 static uint64_t generate_byte_mask(uint8_t mask)
375 {
376 uint64_t r = 0;
377 int i;
378
379 for (i = 0; i < 8; i++) {
380 if ((mask >> i) & 1) {
381 r |= 0xffull << (i * 8);
382 }
383 }
384 return r;
385 }
386
387 static DisasJumpType op_vgbm(DisasContext *s, DisasOps *o)
388 {
389 const uint16_t i2 = get_field(s->fields, i2);
390
391 if (i2 == (i2 & 0xff) * 0x0101) {
392 /*
393 * Masks for both 64 bit elements of the vector are the same.
394 * Trust tcg to produce a good constant loading.
395 */
396 gen_gvec_dup64i(get_field(s->fields, v1),
397 generate_byte_mask(i2 & 0xff));
398 } else {
399 TCGv_i64 t = tcg_temp_new_i64();
400
401 tcg_gen_movi_i64(t, generate_byte_mask(i2 >> 8));
402 write_vec_element_i64(t, get_field(s->fields, v1), 0, ES_64);
403 tcg_gen_movi_i64(t, generate_byte_mask(i2));
404 write_vec_element_i64(t, get_field(s->fields, v1), 1, ES_64);
405 tcg_temp_free_i64(t);
406 }
407 return DISAS_NEXT;
408 }
409
410 static DisasJumpType op_vgm(DisasContext *s, DisasOps *o)
411 {
412 const uint8_t es = get_field(s->fields, m4);
413 const uint8_t bits = NUM_VEC_ELEMENT_BITS(es);
414 const uint8_t i2 = get_field(s->fields, i2) & (bits - 1);
415 const uint8_t i3 = get_field(s->fields, i3) & (bits - 1);
416 uint64_t mask = 0;
417 int i;
418
419 if (es > ES_64) {
420 gen_program_exception(s, PGM_SPECIFICATION);
421 return DISAS_NORETURN;
422 }
423
424 /* generate the mask - take care of wrapping */
425 for (i = i2; ; i = (i + 1) % bits) {
426 mask |= 1ull << (bits - i - 1);
427 if (i == i3) {
428 break;
429 }
430 }
431
432 gen_gvec_dupi(es, get_field(s->fields, v1), mask);
433 return DISAS_NEXT;
434 }
435
436 static DisasJumpType op_vl(DisasContext *s, DisasOps *o)
437 {
438 TCGv_i64 t0 = tcg_temp_new_i64();
439 TCGv_i64 t1 = tcg_temp_new_i64();
440
441 tcg_gen_qemu_ld_i64(t0, o->addr1, get_mem_index(s), MO_TEQ);
442 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
443 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
444 write_vec_element_i64(t0, get_field(s->fields, v1), 0, ES_64);
445 write_vec_element_i64(t1, get_field(s->fields, v1), 1, ES_64);
446 tcg_temp_free(t0);
447 tcg_temp_free(t1);
448 return DISAS_NEXT;
449 }
450
451 static DisasJumpType op_vlr(DisasContext *s, DisasOps *o)
452 {
453 gen_gvec_mov(get_field(s->fields, v1), get_field(s->fields, v2));
454 return DISAS_NEXT;
455 }
456
457 static DisasJumpType op_vlrep(DisasContext *s, DisasOps *o)
458 {
459 const uint8_t es = get_field(s->fields, m3);
460 TCGv_i64 tmp;
461
462 if (es > ES_64) {
463 gen_program_exception(s, PGM_SPECIFICATION);
464 return DISAS_NORETURN;
465 }
466
467 tmp = tcg_temp_new_i64();
468 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
469 gen_gvec_dup_i64(es, get_field(s->fields, v1), tmp);
470 tcg_temp_free_i64(tmp);
471 return DISAS_NEXT;
472 }
473
474 static DisasJumpType op_vle(DisasContext *s, DisasOps *o)
475 {
476 const uint8_t es = s->insn->data;
477 const uint8_t enr = get_field(s->fields, m3);
478 TCGv_i64 tmp;
479
480 if (!valid_vec_element(enr, es)) {
481 gen_program_exception(s, PGM_SPECIFICATION);
482 return DISAS_NORETURN;
483 }
484
485 tmp = tcg_temp_new_i64();
486 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
487 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
488 tcg_temp_free_i64(tmp);
489 return DISAS_NEXT;
490 }
491
492 static DisasJumpType op_vlei(DisasContext *s, DisasOps *o)
493 {
494 const uint8_t es = s->insn->data;
495 const uint8_t enr = get_field(s->fields, m3);
496 TCGv_i64 tmp;
497
498 if (!valid_vec_element(enr, es)) {
499 gen_program_exception(s, PGM_SPECIFICATION);
500 return DISAS_NORETURN;
501 }
502
503 tmp = tcg_const_i64((int16_t)get_field(s->fields, i2));
504 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
505 tcg_temp_free_i64(tmp);
506 return DISAS_NEXT;
507 }
508
509 static DisasJumpType op_vlgv(DisasContext *s, DisasOps *o)
510 {
511 const uint8_t es = get_field(s->fields, m4);
512 TCGv_ptr ptr;
513
514 if (es > ES_64) {
515 gen_program_exception(s, PGM_SPECIFICATION);
516 return DISAS_NORETURN;
517 }
518
519 /* fast path if we don't need the register content */
520 if (!get_field(s->fields, b2)) {
521 uint8_t enr = get_field(s->fields, d2) & (NUM_VEC_ELEMENTS(es) - 1);
522
523 read_vec_element_i64(o->out, get_field(s->fields, v3), enr, es);
524 return DISAS_NEXT;
525 }
526
527 ptr = tcg_temp_new_ptr();
528 get_vec_element_ptr_i64(ptr, get_field(s->fields, v3), o->addr1, es);
529 switch (es) {
530 case ES_8:
531 tcg_gen_ld8u_i64(o->out, ptr, 0);
532 break;
533 case ES_16:
534 tcg_gen_ld16u_i64(o->out, ptr, 0);
535 break;
536 case ES_32:
537 tcg_gen_ld32u_i64(o->out, ptr, 0);
538 break;
539 case ES_64:
540 tcg_gen_ld_i64(o->out, ptr, 0);
541 break;
542 default:
543 g_assert_not_reached();
544 }
545 tcg_temp_free_ptr(ptr);
546
547 return DISAS_NEXT;
548 }
549
550 static DisasJumpType op_vllez(DisasContext *s, DisasOps *o)
551 {
552 uint8_t es = get_field(s->fields, m3);
553 uint8_t enr;
554 TCGv_i64 t;
555
556 switch (es) {
557 /* rightmost sub-element of leftmost doubleword */
558 case ES_8:
559 enr = 7;
560 break;
561 case ES_16:
562 enr = 3;
563 break;
564 case ES_32:
565 enr = 1;
566 break;
567 case ES_64:
568 enr = 0;
569 break;
570 /* leftmost sub-element of leftmost doubleword */
571 case 6:
572 if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
573 es = ES_32;
574 enr = 0;
575 break;
576 }
577 default:
578 /* fallthrough */
579 gen_program_exception(s, PGM_SPECIFICATION);
580 return DISAS_NORETURN;
581 }
582
583 t = tcg_temp_new_i64();
584 tcg_gen_qemu_ld_i64(t, o->addr1, get_mem_index(s), MO_TE | es);
585 zero_vec(get_field(s->fields, v1));
586 write_vec_element_i64(t, get_field(s->fields, v1), enr, es);
587 tcg_temp_free_i64(t);
588 return DISAS_NEXT;
589 }
590
591 static DisasJumpType op_vlm(DisasContext *s, DisasOps *o)
592 {
593 const uint8_t v3 = get_field(s->fields, v3);
594 uint8_t v1 = get_field(s->fields, v1);
595 TCGv_i64 t0, t1;
596
597 if (v3 < v1 || (v3 - v1 + 1) > 16) {
598 gen_program_exception(s, PGM_SPECIFICATION);
599 return DISAS_NORETURN;
600 }
601
602 /*
603 * Check for possible access exceptions by trying to load the last
604 * element. The first element will be checked first next.
605 */
606 t0 = tcg_temp_new_i64();
607 t1 = tcg_temp_new_i64();
608 gen_addi_and_wrap_i64(s, t0, o->addr1, (v3 - v1) * 16 + 8);
609 tcg_gen_qemu_ld_i64(t0, t0, get_mem_index(s), MO_TEQ);
610
611 for (;; v1++) {
612 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
613 write_vec_element_i64(t1, v1, 0, ES_64);
614 if (v1 == v3) {
615 break;
616 }
617 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
618 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
619 write_vec_element_i64(t1, v1, 1, ES_64);
620 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
621 }
622
623 /* Store the last element, loaded first */
624 write_vec_element_i64(t0, v1, 1, ES_64);
625
626 tcg_temp_free_i64(t0);
627 tcg_temp_free_i64(t1);
628 return DISAS_NEXT;
629 }
630
631 static DisasJumpType op_vlbb(DisasContext *s, DisasOps *o)
632 {
633 const int64_t block_size = (1ull << (get_field(s->fields, m3) + 6));
634 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
635 TCGv_ptr a0;
636 TCGv_i64 bytes;
637
638 if (get_field(s->fields, m3) > 6) {
639 gen_program_exception(s, PGM_SPECIFICATION);
640 return DISAS_NORETURN;
641 }
642
643 bytes = tcg_temp_new_i64();
644 a0 = tcg_temp_new_ptr();
645 /* calculate the number of bytes until the next block boundary */
646 tcg_gen_ori_i64(bytes, o->addr1, -block_size);
647 tcg_gen_neg_i64(bytes, bytes);
648
649 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
650 gen_helper_vll(cpu_env, a0, o->addr1, bytes);
651 tcg_temp_free_i64(bytes);
652 tcg_temp_free_ptr(a0);
653 return DISAS_NEXT;
654 }
655
656 static DisasJumpType op_vlvg(DisasContext *s, DisasOps *o)
657 {
658 const uint8_t es = get_field(s->fields, m4);
659 TCGv_ptr ptr;
660
661 if (es > ES_64) {
662 gen_program_exception(s, PGM_SPECIFICATION);
663 return DISAS_NORETURN;
664 }
665
666 /* fast path if we don't need the register content */
667 if (!get_field(s->fields, b2)) {
668 uint8_t enr = get_field(s->fields, d2) & (NUM_VEC_ELEMENTS(es) - 1);
669
670 write_vec_element_i64(o->in2, get_field(s->fields, v1), enr, es);
671 return DISAS_NEXT;
672 }
673
674 ptr = tcg_temp_new_ptr();
675 get_vec_element_ptr_i64(ptr, get_field(s->fields, v1), o->addr1, es);
676 switch (es) {
677 case ES_8:
678 tcg_gen_st8_i64(o->in2, ptr, 0);
679 break;
680 case ES_16:
681 tcg_gen_st16_i64(o->in2, ptr, 0);
682 break;
683 case ES_32:
684 tcg_gen_st32_i64(o->in2, ptr, 0);
685 break;
686 case ES_64:
687 tcg_gen_st_i64(o->in2, ptr, 0);
688 break;
689 default:
690 g_assert_not_reached();
691 }
692 tcg_temp_free_ptr(ptr);
693
694 return DISAS_NEXT;
695 }
696
697 static DisasJumpType op_vlvgp(DisasContext *s, DisasOps *o)
698 {
699 write_vec_element_i64(o->in1, get_field(s->fields, v1), 0, ES_64);
700 write_vec_element_i64(o->in2, get_field(s->fields, v1), 1, ES_64);
701 return DISAS_NEXT;
702 }
703
704 static DisasJumpType op_vll(DisasContext *s, DisasOps *o)
705 {
706 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
707 TCGv_ptr a0 = tcg_temp_new_ptr();
708
709 /* convert highest index into an actual length */
710 tcg_gen_addi_i64(o->in2, o->in2, 1);
711 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
712 gen_helper_vll(cpu_env, a0, o->addr1, o->in2);
713 tcg_temp_free_ptr(a0);
714 return DISAS_NEXT;
715 }
716
717 static DisasJumpType op_vmr(DisasContext *s, DisasOps *o)
718 {
719 const uint8_t v1 = get_field(s->fields, v1);
720 const uint8_t v2 = get_field(s->fields, v2);
721 const uint8_t v3 = get_field(s->fields, v3);
722 const uint8_t es = get_field(s->fields, m4);
723 int dst_idx, src_idx;
724 TCGv_i64 tmp;
725
726 if (es > ES_64) {
727 gen_program_exception(s, PGM_SPECIFICATION);
728 return DISAS_NORETURN;
729 }
730
731 tmp = tcg_temp_new_i64();
732 if (s->fields->op2 == 0x61) {
733 /* iterate backwards to avoid overwriting data we might need later */
734 for (dst_idx = NUM_VEC_ELEMENTS(es) - 1; dst_idx >= 0; dst_idx--) {
735 src_idx = dst_idx / 2;
736 if (dst_idx % 2 == 0) {
737 read_vec_element_i64(tmp, v2, src_idx, es);
738 } else {
739 read_vec_element_i64(tmp, v3, src_idx, es);
740 }
741 write_vec_element_i64(tmp, v1, dst_idx, es);
742 }
743 } else {
744 /* iterate forward to avoid overwriting data we might need later */
745 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(es); dst_idx++) {
746 src_idx = (dst_idx + NUM_VEC_ELEMENTS(es)) / 2;
747 if (dst_idx % 2 == 0) {
748 read_vec_element_i64(tmp, v2, src_idx, es);
749 } else {
750 read_vec_element_i64(tmp, v3, src_idx, es);
751 }
752 write_vec_element_i64(tmp, v1, dst_idx, es);
753 }
754 }
755 tcg_temp_free_i64(tmp);
756 return DISAS_NEXT;
757 }
758
759 static DisasJumpType op_vpk(DisasContext *s, DisasOps *o)
760 {
761 const uint8_t v1 = get_field(s->fields, v1);
762 const uint8_t v2 = get_field(s->fields, v2);
763 const uint8_t v3 = get_field(s->fields, v3);
764 const uint8_t es = get_field(s->fields, m4);
765 static gen_helper_gvec_3 * const vpk[3] = {
766 gen_helper_gvec_vpk16,
767 gen_helper_gvec_vpk32,
768 gen_helper_gvec_vpk64,
769 };
770 static gen_helper_gvec_3 * const vpks[3] = {
771 gen_helper_gvec_vpks16,
772 gen_helper_gvec_vpks32,
773 gen_helper_gvec_vpks64,
774 };
775 static gen_helper_gvec_3_ptr * const vpks_cc[3] = {
776 gen_helper_gvec_vpks_cc16,
777 gen_helper_gvec_vpks_cc32,
778 gen_helper_gvec_vpks_cc64,
779 };
780 static gen_helper_gvec_3 * const vpkls[3] = {
781 gen_helper_gvec_vpkls16,
782 gen_helper_gvec_vpkls32,
783 gen_helper_gvec_vpkls64,
784 };
785 static gen_helper_gvec_3_ptr * const vpkls_cc[3] = {
786 gen_helper_gvec_vpkls_cc16,
787 gen_helper_gvec_vpkls_cc32,
788 gen_helper_gvec_vpkls_cc64,
789 };
790
791 if (es == ES_8 || es > ES_64) {
792 gen_program_exception(s, PGM_SPECIFICATION);
793 return DISAS_NORETURN;
794 }
795
796 switch (s->fields->op2) {
797 case 0x97:
798 if (get_field(s->fields, m5) & 0x1) {
799 gen_gvec_3_ptr(v1, v2, v3, cpu_env, 0, vpks_cc[es - 1]);
800 set_cc_static(s);
801 } else {
802 gen_gvec_3_ool(v1, v2, v3, 0, vpks[es - 1]);
803 }
804 break;
805 case 0x95:
806 if (get_field(s->fields, m5) & 0x1) {
807 gen_gvec_3_ptr(v1, v2, v3, cpu_env, 0, vpkls_cc[es - 1]);
808 set_cc_static(s);
809 } else {
810 gen_gvec_3_ool(v1, v2, v3, 0, vpkls[es - 1]);
811 }
812 break;
813 case 0x94:
814 /* If sources and destination dont't overlap -> fast path */
815 if (v1 != v2 && v1 != v3) {
816 const uint8_t src_es = get_field(s->fields, m4);
817 const uint8_t dst_es = src_es - 1;
818 TCGv_i64 tmp = tcg_temp_new_i64();
819 int dst_idx, src_idx;
820
821 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
822 src_idx = dst_idx;
823 if (src_idx < NUM_VEC_ELEMENTS(src_es)) {
824 read_vec_element_i64(tmp, v2, src_idx, src_es);
825 } else {
826 src_idx -= NUM_VEC_ELEMENTS(src_es);
827 read_vec_element_i64(tmp, v3, src_idx, src_es);
828 }
829 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
830 }
831 tcg_temp_free_i64(tmp);
832 } else {
833 gen_gvec_3_ool(v1, v2, v3, 0, vpk[es - 1]);
834 }
835 break;
836 default:
837 g_assert_not_reached();
838 }
839 return DISAS_NEXT;
840 }
841
842 static DisasJumpType op_vperm(DisasContext *s, DisasOps *o)
843 {
844 gen_gvec_4_ool(get_field(s->fields, v1), get_field(s->fields, v2),
845 get_field(s->fields, v3), get_field(s->fields, v4),
846 0, gen_helper_gvec_vperm);
847 return DISAS_NEXT;
848 }
849
850 static DisasJumpType op_vpdi(DisasContext *s, DisasOps *o)
851 {
852 const uint8_t i2 = extract32(get_field(s->fields, m4), 2, 1);
853 const uint8_t i3 = extract32(get_field(s->fields, m4), 0, 1);
854 TCGv_i64 t0 = tcg_temp_new_i64();
855 TCGv_i64 t1 = tcg_temp_new_i64();
856
857 read_vec_element_i64(t0, get_field(s->fields, v2), i2, ES_64);
858 read_vec_element_i64(t1, get_field(s->fields, v3), i3, ES_64);
859 write_vec_element_i64(t0, get_field(s->fields, v1), 0, ES_64);
860 write_vec_element_i64(t1, get_field(s->fields, v1), 1, ES_64);
861 tcg_temp_free_i64(t0);
862 tcg_temp_free_i64(t1);
863 return DISAS_NEXT;
864 }
865
866 static DisasJumpType op_vrep(DisasContext *s, DisasOps *o)
867 {
868 const uint8_t enr = get_field(s->fields, i2);
869 const uint8_t es = get_field(s->fields, m4);
870
871 if (es > ES_64 || !valid_vec_element(enr, es)) {
872 gen_program_exception(s, PGM_SPECIFICATION);
873 return DISAS_NORETURN;
874 }
875
876 tcg_gen_gvec_dup_mem(es, vec_full_reg_offset(get_field(s->fields, v1)),
877 vec_reg_offset(get_field(s->fields, v3), enr, es),
878 16, 16);
879 return DISAS_NEXT;
880 }
881
882 static DisasJumpType op_vrepi(DisasContext *s, DisasOps *o)
883 {
884 const int64_t data = (int16_t)get_field(s->fields, i2);
885 const uint8_t es = get_field(s->fields, m3);
886
887 if (es > ES_64) {
888 gen_program_exception(s, PGM_SPECIFICATION);
889 return DISAS_NORETURN;
890 }
891
892 gen_gvec_dupi(es, get_field(s->fields, v1), data);
893 return DISAS_NEXT;
894 }
895
896 static DisasJumpType op_vsce(DisasContext *s, DisasOps *o)
897 {
898 const uint8_t es = s->insn->data;
899 const uint8_t enr = get_field(s->fields, m3);
900 TCGv_i64 tmp;
901
902 if (!valid_vec_element(enr, es)) {
903 gen_program_exception(s, PGM_SPECIFICATION);
904 return DISAS_NORETURN;
905 }
906
907 tmp = tcg_temp_new_i64();
908 read_vec_element_i64(tmp, get_field(s->fields, v2), enr, es);
909 tcg_gen_add_i64(o->addr1, o->addr1, tmp);
910 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
911
912 read_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
913 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
914 tcg_temp_free_i64(tmp);
915 return DISAS_NEXT;
916 }
917
918 static void gen_sel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c)
919 {
920 TCGv_i64 t = tcg_temp_new_i64();
921
922 /* bit in c not set -> copy bit from b */
923 tcg_gen_andc_i64(t, b, c);
924 /* bit in c set -> copy bit from a */
925 tcg_gen_and_i64(d, a, c);
926 /* merge the results */
927 tcg_gen_or_i64(d, d, t);
928 tcg_temp_free_i64(t);
929 }
930
931 static void gen_sel_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b,
932 TCGv_vec c)
933 {
934 TCGv_vec t = tcg_temp_new_vec_matching(d);
935
936 tcg_gen_andc_vec(vece, t, b, c);
937 tcg_gen_and_vec(vece, d, a, c);
938 tcg_gen_or_vec(vece, d, d, t);
939 tcg_temp_free_vec(t);
940 }
941
942 static DisasJumpType op_vsel(DisasContext *s, DisasOps *o)
943 {
944 static const GVecGen4 gvec_op = {
945 .fni8 = gen_sel_i64,
946 .fniv = gen_sel_vec,
947 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
948 };
949
950 gen_gvec_4(get_field(s->fields, v1), get_field(s->fields, v2),
951 get_field(s->fields, v3), get_field(s->fields, v4), &gvec_op);
952 return DISAS_NEXT;
953 }
954
955 static DisasJumpType op_vseg(DisasContext *s, DisasOps *o)
956 {
957 const uint8_t es = get_field(s->fields, m3);
958 int idx1, idx2;
959 TCGv_i64 tmp;
960
961 switch (es) {
962 case ES_8:
963 idx1 = 7;
964 idx2 = 15;
965 break;
966 case ES_16:
967 idx1 = 3;
968 idx2 = 7;
969 break;
970 case ES_32:
971 idx1 = 1;
972 idx2 = 3;
973 break;
974 default:
975 gen_program_exception(s, PGM_SPECIFICATION);
976 return DISAS_NORETURN;
977 }
978
979 tmp = tcg_temp_new_i64();
980 read_vec_element_i64(tmp, get_field(s->fields, v2), idx1, es | MO_SIGN);
981 write_vec_element_i64(tmp, get_field(s->fields, v1), 0, ES_64);
982 read_vec_element_i64(tmp, get_field(s->fields, v2), idx2, es | MO_SIGN);
983 write_vec_element_i64(tmp, get_field(s->fields, v1), 1, ES_64);
984 tcg_temp_free_i64(tmp);
985 return DISAS_NEXT;
986 }
987
988 static DisasJumpType op_vst(DisasContext *s, DisasOps *o)
989 {
990 TCGv_i64 tmp = tcg_const_i64(16);
991
992 /* Probe write access before actually modifying memory */
993 gen_helper_probe_write_access(cpu_env, o->addr1, tmp);
994
995 read_vec_element_i64(tmp, get_field(s->fields, v1), 0, ES_64);
996 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
997 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
998 read_vec_element_i64(tmp, get_field(s->fields, v1), 1, ES_64);
999 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1000 tcg_temp_free_i64(tmp);
1001 return DISAS_NEXT;
1002 }
1003
1004 static DisasJumpType op_vste(DisasContext *s, DisasOps *o)
1005 {
1006 const uint8_t es = s->insn->data;
1007 const uint8_t enr = get_field(s->fields, m3);
1008 TCGv_i64 tmp;
1009
1010 if (!valid_vec_element(enr, es)) {
1011 gen_program_exception(s, PGM_SPECIFICATION);
1012 return DISAS_NORETURN;
1013 }
1014
1015 tmp = tcg_temp_new_i64();
1016 read_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
1017 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
1018 tcg_temp_free_i64(tmp);
1019 return DISAS_NEXT;
1020 }
1021
1022 static DisasJumpType op_vstm(DisasContext *s, DisasOps *o)
1023 {
1024 const uint8_t v3 = get_field(s->fields, v3);
1025 uint8_t v1 = get_field(s->fields, v1);
1026 TCGv_i64 tmp;
1027
1028 while (v3 < v1 || (v3 - v1 + 1) > 16) {
1029 gen_program_exception(s, PGM_SPECIFICATION);
1030 return DISAS_NORETURN;
1031 }
1032
1033 /* Probe write access before actually modifying memory */
1034 tmp = tcg_const_i64((v3 - v1 + 1) * 16);
1035 gen_helper_probe_write_access(cpu_env, o->addr1, tmp);
1036
1037 for (;; v1++) {
1038 read_vec_element_i64(tmp, v1, 0, ES_64);
1039 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1040 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1041 read_vec_element_i64(tmp, v1, 1, ES_64);
1042 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1043 if (v1 == v3) {
1044 break;
1045 }
1046 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1047 }
1048 tcg_temp_free_i64(tmp);
1049 return DISAS_NEXT;
1050 }
1051
1052 static DisasJumpType op_vstl(DisasContext *s, DisasOps *o)
1053 {
1054 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
1055 TCGv_ptr a0 = tcg_temp_new_ptr();
1056
1057 /* convert highest index into an actual length */
1058 tcg_gen_addi_i64(o->in2, o->in2, 1);
1059 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
1060 gen_helper_vstl(cpu_env, a0, o->addr1, o->in2);
1061 tcg_temp_free_ptr(a0);
1062 return DISAS_NEXT;
1063 }
1064
1065 static DisasJumpType op_vup(DisasContext *s, DisasOps *o)
1066 {
1067 const bool logical = s->fields->op2 == 0xd4 || s->fields->op2 == 0xd5;
1068 const uint8_t v1 = get_field(s->fields, v1);
1069 const uint8_t v2 = get_field(s->fields, v2);
1070 const uint8_t src_es = get_field(s->fields, m3);
1071 const uint8_t dst_es = src_es + 1;
1072 int dst_idx, src_idx;
1073 TCGv_i64 tmp;
1074
1075 if (src_es > ES_32) {
1076 gen_program_exception(s, PGM_SPECIFICATION);
1077 return DISAS_NORETURN;
1078 }
1079
1080 tmp = tcg_temp_new_i64();
1081 if (s->fields->op2 == 0xd7 || s->fields->op2 == 0xd5) {
1082 /* iterate backwards to avoid overwriting data we might need later */
1083 for (dst_idx = NUM_VEC_ELEMENTS(dst_es) - 1; dst_idx >= 0; dst_idx--) {
1084 src_idx = dst_idx;
1085 read_vec_element_i64(tmp, v2, src_idx,
1086 src_es | (logical ? 0 : MO_SIGN));
1087 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1088 }
1089
1090 } else {
1091 /* iterate forward to avoid overwriting data we might need later */
1092 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
1093 src_idx = dst_idx + NUM_VEC_ELEMENTS(src_es) / 2;
1094 read_vec_element_i64(tmp, v2, src_idx,
1095 src_es | (logical ? 0 : MO_SIGN));
1096 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1097 }
1098 }
1099 tcg_temp_free_i64(tmp);
1100 return DISAS_NEXT;
1101 }
1102
1103 static DisasJumpType op_va(DisasContext *s, DisasOps *o)
1104 {
1105 const uint8_t es = get_field(s->fields, m4);
1106
1107 if (es > ES_128) {
1108 gen_program_exception(s, PGM_SPECIFICATION);
1109 return DISAS_NORETURN;
1110 } else if (es == ES_128) {
1111 gen_gvec128_3_i64(tcg_gen_add2_i64, get_field(s->fields, v1),
1112 get_field(s->fields, v2), get_field(s->fields, v3));
1113 return DISAS_NEXT;
1114 }
1115 gen_gvec_fn_3(add, es, get_field(s->fields, v1), get_field(s->fields, v2),
1116 get_field(s->fields, v3));
1117 return DISAS_NEXT;
1118 }
1119
1120 static void gen_acc(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, uint8_t es)
1121 {
1122 const uint8_t msb_bit_nr = NUM_VEC_ELEMENT_BITS(es) - 1;
1123 TCGv_i64 msb_mask = tcg_const_i64(dup_const(es, 1ull << msb_bit_nr));
1124 TCGv_i64 t1 = tcg_temp_new_i64();
1125 TCGv_i64 t2 = tcg_temp_new_i64();
1126 TCGv_i64 t3 = tcg_temp_new_i64();
1127
1128 /* Calculate the carry into the MSB, ignoring the old MSBs */
1129 tcg_gen_andc_i64(t1, a, msb_mask);
1130 tcg_gen_andc_i64(t2, b, msb_mask);
1131 tcg_gen_add_i64(t1, t1, t2);
1132 /* Calculate the MSB without any carry into it */
1133 tcg_gen_xor_i64(t3, a, b);
1134 /* Calculate the carry out of the MSB in the MSB bit position */
1135 tcg_gen_and_i64(d, a, b);
1136 tcg_gen_and_i64(t1, t1, t3);
1137 tcg_gen_or_i64(d, d, t1);
1138 /* Isolate and shift the carry into position */
1139 tcg_gen_and_i64(d, d, msb_mask);
1140 tcg_gen_shri_i64(d, d, msb_bit_nr);
1141
1142 tcg_temp_free_i64(t1);
1143 tcg_temp_free_i64(t2);
1144 tcg_temp_free_i64(t3);
1145 }
1146
1147 static void gen_acc8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1148 {
1149 gen_acc(d, a, b, ES_8);
1150 }
1151
1152 static void gen_acc16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1153 {
1154 gen_acc(d, a, b, ES_16);
1155 }
1156
1157 static void gen_acc_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1158 {
1159 TCGv_i32 t = tcg_temp_new_i32();
1160
1161 tcg_gen_add_i32(t, a, b);
1162 tcg_gen_setcond_i32(TCG_COND_LTU, d, t, b);
1163 tcg_temp_free_i32(t);
1164 }
1165
1166 static void gen_acc_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1167 {
1168 TCGv_i64 t = tcg_temp_new_i64();
1169
1170 tcg_gen_add_i64(t, a, b);
1171 tcg_gen_setcond_i64(TCG_COND_LTU, d, t, b);
1172 tcg_temp_free_i64(t);
1173 }
1174
1175 static void gen_acc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
1176 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
1177 {
1178 TCGv_i64 th = tcg_temp_new_i64();
1179 TCGv_i64 tl = tcg_temp_new_i64();
1180 TCGv_i64 zero = tcg_const_i64(0);
1181
1182 tcg_gen_add2_i64(tl, th, al, zero, bl, zero);
1183 tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1184 tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1185 tcg_gen_mov_i64(dh, zero);
1186
1187 tcg_temp_free_i64(th);
1188 tcg_temp_free_i64(tl);
1189 tcg_temp_free_i64(zero);
1190 }
1191
1192 static DisasJumpType op_vacc(DisasContext *s, DisasOps *o)
1193 {
1194 const uint8_t es = get_field(s->fields, m4);
1195 static const GVecGen3 g[4] = {
1196 { .fni8 = gen_acc8_i64, },
1197 { .fni8 = gen_acc16_i64, },
1198 { .fni4 = gen_acc_i32, },
1199 { .fni8 = gen_acc_i64, },
1200 };
1201
1202 if (es > ES_128) {
1203 gen_program_exception(s, PGM_SPECIFICATION);
1204 return DISAS_NORETURN;
1205 } else if (es == ES_128) {
1206 gen_gvec128_3_i64(gen_acc2_i64, get_field(s->fields, v1),
1207 get_field(s->fields, v2), get_field(s->fields, v3));
1208 return DISAS_NEXT;
1209 }
1210 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1211 get_field(s->fields, v3), &g[es]);
1212 return DISAS_NEXT;
1213 }
1214
1215 static void gen_ac2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1216 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1217 {
1218 TCGv_i64 tl = tcg_temp_new_i64();
1219 TCGv_i64 th = tcg_const_i64(0);
1220
1221 /* extract the carry only */
1222 tcg_gen_extract_i64(tl, cl, 0, 1);
1223 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1224 tcg_gen_add2_i64(dl, dh, dl, dh, tl, th);
1225
1226 tcg_temp_free_i64(tl);
1227 tcg_temp_free_i64(th);
1228 }
1229
1230 static DisasJumpType op_vac(DisasContext *s, DisasOps *o)
1231 {
1232 if (get_field(s->fields, m5) != ES_128) {
1233 gen_program_exception(s, PGM_SPECIFICATION);
1234 return DISAS_NORETURN;
1235 }
1236
1237 gen_gvec128_4_i64(gen_ac2_i64, get_field(s->fields, v1),
1238 get_field(s->fields, v2), get_field(s->fields, v3),
1239 get_field(s->fields, v4));
1240 return DISAS_NEXT;
1241 }
1242
1243 static void gen_accc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1244 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1245 {
1246 TCGv_i64 tl = tcg_temp_new_i64();
1247 TCGv_i64 th = tcg_temp_new_i64();
1248 TCGv_i64 zero = tcg_const_i64(0);
1249
1250 tcg_gen_andi_i64(tl, cl, 1);
1251 tcg_gen_add2_i64(tl, th, tl, zero, al, zero);
1252 tcg_gen_add2_i64(tl, th, tl, th, bl, zero);
1253 tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1254 tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1255 tcg_gen_mov_i64(dh, zero);
1256
1257 tcg_temp_free_i64(tl);
1258 tcg_temp_free_i64(th);
1259 tcg_temp_free_i64(zero);
1260 }
1261
1262 static DisasJumpType op_vaccc(DisasContext *s, DisasOps *o)
1263 {
1264 if (get_field(s->fields, m5) != ES_128) {
1265 gen_program_exception(s, PGM_SPECIFICATION);
1266 return DISAS_NORETURN;
1267 }
1268
1269 gen_gvec128_4_i64(gen_accc2_i64, get_field(s->fields, v1),
1270 get_field(s->fields, v2), get_field(s->fields, v3),
1271 get_field(s->fields, v4));
1272 return DISAS_NEXT;
1273 }
1274
1275 static DisasJumpType op_vn(DisasContext *s, DisasOps *o)
1276 {
1277 gen_gvec_fn_3(and, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1278 get_field(s->fields, v3));
1279 return DISAS_NEXT;
1280 }
1281
1282 static DisasJumpType op_vnc(DisasContext *s, DisasOps *o)
1283 {
1284 gen_gvec_fn_3(andc, ES_8, get_field(s->fields, v1),
1285 get_field(s->fields, v2), get_field(s->fields, v3));
1286 return DISAS_NEXT;
1287 }
1288
1289 static void gen_avg_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1290 {
1291 TCGv_i64 t0 = tcg_temp_new_i64();
1292 TCGv_i64 t1 = tcg_temp_new_i64();
1293
1294 tcg_gen_ext_i32_i64(t0, a);
1295 tcg_gen_ext_i32_i64(t1, b);
1296 tcg_gen_add_i64(t0, t0, t1);
1297 tcg_gen_addi_i64(t0, t0, 1);
1298 tcg_gen_shri_i64(t0, t0, 1);
1299 tcg_gen_extrl_i64_i32(d, t0);
1300
1301 tcg_temp_free(t0);
1302 tcg_temp_free(t1);
1303 }
1304
1305 static void gen_avg_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1306 {
1307 TCGv_i64 dh = tcg_temp_new_i64();
1308 TCGv_i64 ah = tcg_temp_new_i64();
1309 TCGv_i64 bh = tcg_temp_new_i64();
1310
1311 /* extending the sign by one bit is sufficient */
1312 tcg_gen_extract_i64(ah, al, 63, 1);
1313 tcg_gen_extract_i64(bh, bl, 63, 1);
1314 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1315 gen_addi2_i64(dl, dh, dl, dh, 1);
1316 tcg_gen_extract2_i64(dl, dl, dh, 1);
1317
1318 tcg_temp_free_i64(dh);
1319 tcg_temp_free_i64(ah);
1320 tcg_temp_free_i64(bh);
1321 }
1322
1323 static DisasJumpType op_vavg(DisasContext *s, DisasOps *o)
1324 {
1325 const uint8_t es = get_field(s->fields, m4);
1326 static const GVecGen3 g[4] = {
1327 { .fno = gen_helper_gvec_vavg8, },
1328 { .fno = gen_helper_gvec_vavg16, },
1329 { .fni4 = gen_avg_i32, },
1330 { .fni8 = gen_avg_i64, },
1331 };
1332
1333 if (es > ES_64) {
1334 gen_program_exception(s, PGM_SPECIFICATION);
1335 return DISAS_NORETURN;
1336 }
1337 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1338 get_field(s->fields, v3), &g[es]);
1339 return DISAS_NEXT;
1340 }
1341
1342 static void gen_avgl_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1343 {
1344 TCGv_i64 t0 = tcg_temp_new_i64();
1345 TCGv_i64 t1 = tcg_temp_new_i64();
1346
1347 tcg_gen_extu_i32_i64(t0, a);
1348 tcg_gen_extu_i32_i64(t1, b);
1349 tcg_gen_add_i64(t0, t0, t1);
1350 tcg_gen_addi_i64(t0, t0, 1);
1351 tcg_gen_shri_i64(t0, t0, 1);
1352 tcg_gen_extrl_i64_i32(d, t0);
1353
1354 tcg_temp_free(t0);
1355 tcg_temp_free(t1);
1356 }
1357
1358 static void gen_avgl_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1359 {
1360 TCGv_i64 dh = tcg_temp_new_i64();
1361 TCGv_i64 zero = tcg_const_i64(0);
1362
1363 tcg_gen_add2_i64(dl, dh, al, zero, bl, zero);
1364 gen_addi2_i64(dl, dh, dl, dh, 1);
1365 tcg_gen_extract2_i64(dl, dl, dh, 1);
1366
1367 tcg_temp_free_i64(dh);
1368 tcg_temp_free_i64(zero);
1369 }
1370
1371 static DisasJumpType op_vavgl(DisasContext *s, DisasOps *o)
1372 {
1373 const uint8_t es = get_field(s->fields, m4);
1374 static const GVecGen3 g[4] = {
1375 { .fno = gen_helper_gvec_vavgl8, },
1376 { .fno = gen_helper_gvec_vavgl16, },
1377 { .fni4 = gen_avgl_i32, },
1378 { .fni8 = gen_avgl_i64, },
1379 };
1380
1381 if (es > ES_64) {
1382 gen_program_exception(s, PGM_SPECIFICATION);
1383 return DISAS_NORETURN;
1384 }
1385 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1386 get_field(s->fields, v3), &g[es]);
1387 return DISAS_NEXT;
1388 }
1389
1390 static DisasJumpType op_vcksm(DisasContext *s, DisasOps *o)
1391 {
1392 TCGv_i32 tmp = tcg_temp_new_i32();
1393 TCGv_i32 sum = tcg_temp_new_i32();
1394 int i;
1395
1396 read_vec_element_i32(sum, get_field(s->fields, v3), 1, ES_32);
1397 for (i = 0; i < 4; i++) {
1398 read_vec_element_i32(tmp, get_field(s->fields, v2), i, ES_32);
1399 tcg_gen_add2_i32(tmp, sum, sum, sum, tmp, tmp);
1400 }
1401 zero_vec(get_field(s->fields, v1));
1402 write_vec_element_i32(sum, get_field(s->fields, v1), 1, ES_32);
1403
1404 tcg_temp_free_i32(tmp);
1405 tcg_temp_free_i32(sum);
1406 return DISAS_NEXT;
1407 }
1408
1409 static DisasJumpType op_vec(DisasContext *s, DisasOps *o)
1410 {
1411 uint8_t es = get_field(s->fields, m3);
1412 const uint8_t enr = NUM_VEC_ELEMENTS(es) / 2 - 1;
1413
1414 if (es > ES_64) {
1415 gen_program_exception(s, PGM_SPECIFICATION);
1416 return DISAS_NORETURN;
1417 }
1418 if (s->fields->op2 == 0xdb) {
1419 es |= MO_SIGN;
1420 }
1421
1422 o->in1 = tcg_temp_new_i64();
1423 o->in2 = tcg_temp_new_i64();
1424 read_vec_element_i64(o->in1, get_field(s->fields, v1), enr, es);
1425 read_vec_element_i64(o->in2, get_field(s->fields, v2), enr, es);
1426 return DISAS_NEXT;
1427 }
1428
1429 static DisasJumpType op_vc(DisasContext *s, DisasOps *o)
1430 {
1431 const uint8_t es = get_field(s->fields, m4);
1432 TCGCond cond = s->insn->data;
1433
1434 if (es > ES_64) {
1435 gen_program_exception(s, PGM_SPECIFICATION);
1436 return DISAS_NORETURN;
1437 }
1438
1439 tcg_gen_gvec_cmp(cond, es,
1440 vec_full_reg_offset(get_field(s->fields, v1)),
1441 vec_full_reg_offset(get_field(s->fields, v2)),
1442 vec_full_reg_offset(get_field(s->fields, v3)), 16, 16);
1443 if (get_field(s->fields, m5) & 0x1) {
1444 TCGv_i64 low = tcg_temp_new_i64();
1445 TCGv_i64 high = tcg_temp_new_i64();
1446
1447 read_vec_element_i64(high, get_field(s->fields, v1), 0, ES_64);
1448 read_vec_element_i64(low, get_field(s->fields, v1), 1, ES_64);
1449 gen_op_update2_cc_i64(s, CC_OP_VC, low, high);
1450
1451 tcg_temp_free_i64(low);
1452 tcg_temp_free_i64(high);
1453 }
1454 return DISAS_NEXT;
1455 }
1456
1457 static void gen_clz_i32(TCGv_i32 d, TCGv_i32 a)
1458 {
1459 tcg_gen_clzi_i32(d, a, 32);
1460 }
1461
1462 static void gen_clz_i64(TCGv_i64 d, TCGv_i64 a)
1463 {
1464 tcg_gen_clzi_i64(d, a, 64);
1465 }
1466
1467 static DisasJumpType op_vclz(DisasContext *s, DisasOps *o)
1468 {
1469 const uint8_t es = get_field(s->fields, m3);
1470 static const GVecGen2 g[4] = {
1471 { .fno = gen_helper_gvec_vclz8, },
1472 { .fno = gen_helper_gvec_vclz16, },
1473 { .fni4 = gen_clz_i32, },
1474 { .fni8 = gen_clz_i64, },
1475 };
1476
1477 if (es > ES_64) {
1478 gen_program_exception(s, PGM_SPECIFICATION);
1479 return DISAS_NORETURN;
1480 }
1481 gen_gvec_2(get_field(s->fields, v1), get_field(s->fields, v2), &g[es]);
1482 return DISAS_NEXT;
1483 }
1484
1485 static void gen_ctz_i32(TCGv_i32 d, TCGv_i32 a)
1486 {
1487 tcg_gen_ctzi_i32(d, a, 32);
1488 }
1489
1490 static void gen_ctz_i64(TCGv_i64 d, TCGv_i64 a)
1491 {
1492 tcg_gen_ctzi_i64(d, a, 64);
1493 }
1494
1495 static DisasJumpType op_vctz(DisasContext *s, DisasOps *o)
1496 {
1497 const uint8_t es = get_field(s->fields, m3);
1498 static const GVecGen2 g[4] = {
1499 { .fno = gen_helper_gvec_vctz8, },
1500 { .fno = gen_helper_gvec_vctz16, },
1501 { .fni4 = gen_ctz_i32, },
1502 { .fni8 = gen_ctz_i64, },
1503 };
1504
1505 if (es > ES_64) {
1506 gen_program_exception(s, PGM_SPECIFICATION);
1507 return DISAS_NORETURN;
1508 }
1509 gen_gvec_2(get_field(s->fields, v1), get_field(s->fields, v2), &g[es]);
1510 return DISAS_NEXT;
1511 }
1512
1513 static DisasJumpType op_vx(DisasContext *s, DisasOps *o)
1514 {
1515 gen_gvec_fn_3(xor, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1516 get_field(s->fields, v3));
1517 return DISAS_NEXT;
1518 }
1519
1520 static DisasJumpType op_vgfm(DisasContext *s, DisasOps *o)
1521 {
1522 const uint8_t es = get_field(s->fields, m4);
1523 static const GVecGen3 g[4] = {
1524 { .fno = gen_helper_gvec_vgfm8, },
1525 { .fno = gen_helper_gvec_vgfm16, },
1526 { .fno = gen_helper_gvec_vgfm32, },
1527 { .fno = gen_helper_gvec_vgfm64, },
1528 };
1529
1530 if (es > ES_64) {
1531 gen_program_exception(s, PGM_SPECIFICATION);
1532 return DISAS_NORETURN;
1533 }
1534 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1535 get_field(s->fields, v3), &g[es]);
1536 return DISAS_NEXT;
1537 }
1538
1539 static DisasJumpType op_vgfma(DisasContext *s, DisasOps *o)
1540 {
1541 const uint8_t es = get_field(s->fields, m5);
1542 static const GVecGen4 g[4] = {
1543 { .fno = gen_helper_gvec_vgfma8, },
1544 { .fno = gen_helper_gvec_vgfma16, },
1545 { .fno = gen_helper_gvec_vgfma32, },
1546 { .fno = gen_helper_gvec_vgfma64, },
1547 };
1548
1549 if (es > ES_64) {
1550 gen_program_exception(s, PGM_SPECIFICATION);
1551 return DISAS_NORETURN;
1552 }
1553 gen_gvec_4(get_field(s->fields, v1), get_field(s->fields, v2),
1554 get_field(s->fields, v3), get_field(s->fields, v4), &g[es]);
1555 return DISAS_NEXT;
1556 }
1557
1558 static DisasJumpType op_vlc(DisasContext *s, DisasOps *o)
1559 {
1560 const uint8_t es = get_field(s->fields, m3);
1561
1562 if (es > ES_64) {
1563 gen_program_exception(s, PGM_SPECIFICATION);
1564 return DISAS_NORETURN;
1565 }
1566
1567 gen_gvec_fn_2(neg, es, get_field(s->fields, v1), get_field(s->fields, v2));
1568 return DISAS_NEXT;
1569 }
1570
1571 static DisasJumpType op_vlp(DisasContext *s, DisasOps *o)
1572 {
1573 const uint8_t es = get_field(s->fields, m3);
1574
1575 if (es > ES_64) {
1576 gen_program_exception(s, PGM_SPECIFICATION);
1577 return DISAS_NORETURN;
1578 }
1579
1580 gen_gvec_fn_2(abs, es, get_field(s->fields, v1), get_field(s->fields, v2));
1581 return DISAS_NEXT;
1582 }
1583
1584 static DisasJumpType op_vmx(DisasContext *s, DisasOps *o)
1585 {
1586 const uint8_t v1 = get_field(s->fields, v1);
1587 const uint8_t v2 = get_field(s->fields, v2);
1588 const uint8_t v3 = get_field(s->fields, v3);
1589 const uint8_t es = get_field(s->fields, m4);
1590
1591 if (es > ES_64) {
1592 gen_program_exception(s, PGM_SPECIFICATION);
1593 return DISAS_NORETURN;
1594 }
1595
1596 switch (s->fields->op2) {
1597 case 0xff:
1598 gen_gvec_fn_3(smax, es, v1, v2, v3);
1599 break;
1600 case 0xfd:
1601 gen_gvec_fn_3(umax, es, v1, v2, v3);
1602 break;
1603 case 0xfe:
1604 gen_gvec_fn_3(smin, es, v1, v2, v3);
1605 break;
1606 case 0xfc:
1607 gen_gvec_fn_3(umin, es, v1, v2, v3);
1608 break;
1609 default:
1610 g_assert_not_reached();
1611 }
1612 return DISAS_NEXT;
1613 }
1614
1615 static void gen_mal_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1616 {
1617 TCGv_i32 t0 = tcg_temp_new_i32();
1618
1619 tcg_gen_mul_i32(t0, a, b);
1620 tcg_gen_add_i32(d, t0, c);
1621
1622 tcg_temp_free_i32(t0);
1623 }
1624
1625 static void gen_mah_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1626 {
1627 TCGv_i64 t0 = tcg_temp_new_i64();
1628 TCGv_i64 t1 = tcg_temp_new_i64();
1629 TCGv_i64 t2 = tcg_temp_new_i64();
1630
1631 tcg_gen_ext_i32_i64(t0, a);
1632 tcg_gen_ext_i32_i64(t1, b);
1633 tcg_gen_ext_i32_i64(t2, c);
1634 tcg_gen_mul_i64(t0, t0, t1);
1635 tcg_gen_add_i64(t0, t0, t2);
1636 tcg_gen_extrh_i64_i32(d, t0);
1637
1638 tcg_temp_free(t0);
1639 tcg_temp_free(t1);
1640 tcg_temp_free(t2);
1641 }
1642
1643 static void gen_malh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1644 {
1645 TCGv_i64 t0 = tcg_temp_new_i64();
1646 TCGv_i64 t1 = tcg_temp_new_i64();
1647 TCGv_i64 t2 = tcg_temp_new_i64();
1648
1649 tcg_gen_extu_i32_i64(t0, a);
1650 tcg_gen_extu_i32_i64(t1, b);
1651 tcg_gen_extu_i32_i64(t2, c);
1652 tcg_gen_mul_i64(t0, t0, t1);
1653 tcg_gen_add_i64(t0, t0, t2);
1654 tcg_gen_extrh_i64_i32(d, t0);
1655
1656 tcg_temp_free(t0);
1657 tcg_temp_free(t1);
1658 tcg_temp_free(t2);
1659 }
1660
1661 static DisasJumpType op_vma(DisasContext *s, DisasOps *o)
1662 {
1663 const uint8_t es = get_field(s->fields, m5);
1664 static const GVecGen4 g_vmal[3] = {
1665 { .fno = gen_helper_gvec_vmal8, },
1666 { .fno = gen_helper_gvec_vmal16, },
1667 { .fni4 = gen_mal_i32, },
1668 };
1669 static const GVecGen4 g_vmah[3] = {
1670 { .fno = gen_helper_gvec_vmah8, },
1671 { .fno = gen_helper_gvec_vmah16, },
1672 { .fni4 = gen_mah_i32, },
1673 };
1674 static const GVecGen4 g_vmalh[3] = {
1675 { .fno = gen_helper_gvec_vmalh8, },
1676 { .fno = gen_helper_gvec_vmalh16, },
1677 { .fni4 = gen_malh_i32, },
1678 };
1679 static const GVecGen4 g_vmae[3] = {
1680 { .fno = gen_helper_gvec_vmae8, },
1681 { .fno = gen_helper_gvec_vmae16, },
1682 { .fno = gen_helper_gvec_vmae32, },
1683 };
1684 static const GVecGen4 g_vmale[3] = {
1685 { .fno = gen_helper_gvec_vmale8, },
1686 { .fno = gen_helper_gvec_vmale16, },
1687 { .fno = gen_helper_gvec_vmale32, },
1688 };
1689 static const GVecGen4 g_vmao[3] = {
1690 { .fno = gen_helper_gvec_vmao8, },
1691 { .fno = gen_helper_gvec_vmao16, },
1692 { .fno = gen_helper_gvec_vmao32, },
1693 };
1694 static const GVecGen4 g_vmalo[3] = {
1695 { .fno = gen_helper_gvec_vmalo8, },
1696 { .fno = gen_helper_gvec_vmalo16, },
1697 { .fno = gen_helper_gvec_vmalo32, },
1698 };
1699 const GVecGen4 *fn;
1700
1701 if (es > ES_32) {
1702 gen_program_exception(s, PGM_SPECIFICATION);
1703 return DISAS_NORETURN;
1704 }
1705
1706 switch (s->fields->op2) {
1707 case 0xaa:
1708 fn = &g_vmal[es];
1709 break;
1710 case 0xab:
1711 fn = &g_vmah[es];
1712 break;
1713 case 0xa9:
1714 fn = &g_vmalh[es];
1715 break;
1716 case 0xae:
1717 fn = &g_vmae[es];
1718 break;
1719 case 0xac:
1720 fn = &g_vmale[es];
1721 break;
1722 case 0xaf:
1723 fn = &g_vmao[es];
1724 break;
1725 case 0xad:
1726 fn = &g_vmalo[es];
1727 break;
1728 default:
1729 g_assert_not_reached();
1730 }
1731
1732 gen_gvec_4(get_field(s->fields, v1), get_field(s->fields, v2),
1733 get_field(s->fields, v3), get_field(s->fields, v4), fn);
1734 return DISAS_NEXT;
1735 }
1736
1737 static void gen_mh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1738 {
1739 TCGv_i32 t = tcg_temp_new_i32();
1740
1741 tcg_gen_muls2_i32(t, d, a, b);
1742 tcg_temp_free_i32(t);
1743 }
1744
1745 static void gen_mlh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1746 {
1747 TCGv_i32 t = tcg_temp_new_i32();
1748
1749 tcg_gen_mulu2_i32(t, d, a, b);
1750 tcg_temp_free_i32(t);
1751 }
1752
1753 static DisasJumpType op_vm(DisasContext *s, DisasOps *o)
1754 {
1755 const uint8_t es = get_field(s->fields, m4);
1756 static const GVecGen3 g_vmh[3] = {
1757 { .fno = gen_helper_gvec_vmh8, },
1758 { .fno = gen_helper_gvec_vmh16, },
1759 { .fni4 = gen_mh_i32, },
1760 };
1761 static const GVecGen3 g_vmlh[3] = {
1762 { .fno = gen_helper_gvec_vmlh8, },
1763 { .fno = gen_helper_gvec_vmlh16, },
1764 { .fni4 = gen_mlh_i32, },
1765 };
1766 static const GVecGen3 g_vme[3] = {
1767 { .fno = gen_helper_gvec_vme8, },
1768 { .fno = gen_helper_gvec_vme16, },
1769 { .fno = gen_helper_gvec_vme32, },
1770 };
1771 static const GVecGen3 g_vmle[3] = {
1772 { .fno = gen_helper_gvec_vmle8, },
1773 { .fno = gen_helper_gvec_vmle16, },
1774 { .fno = gen_helper_gvec_vmle32, },
1775 };
1776 static const GVecGen3 g_vmo[3] = {
1777 { .fno = gen_helper_gvec_vmo8, },
1778 { .fno = gen_helper_gvec_vmo16, },
1779 { .fno = gen_helper_gvec_vmo32, },
1780 };
1781 static const GVecGen3 g_vmlo[3] = {
1782 { .fno = gen_helper_gvec_vmlo8, },
1783 { .fno = gen_helper_gvec_vmlo16, },
1784 { .fno = gen_helper_gvec_vmlo32, },
1785 };
1786 const GVecGen3 *fn;
1787
1788 if (es > ES_32) {
1789 gen_program_exception(s, PGM_SPECIFICATION);
1790 return DISAS_NORETURN;
1791 }
1792
1793 switch (s->fields->op2) {
1794 case 0xa2:
1795 gen_gvec_fn_3(mul, es, get_field(s->fields, v1),
1796 get_field(s->fields, v2), get_field(s->fields, v3));
1797 return DISAS_NEXT;
1798 case 0xa3:
1799 fn = &g_vmh[es];
1800 break;
1801 case 0xa1:
1802 fn = &g_vmlh[es];
1803 break;
1804 case 0xa6:
1805 fn = &g_vme[es];
1806 break;
1807 case 0xa4:
1808 fn = &g_vmle[es];
1809 break;
1810 case 0xa7:
1811 fn = &g_vmo[es];
1812 break;
1813 case 0xa5:
1814 fn = &g_vmlo[es];
1815 break;
1816 default:
1817 g_assert_not_reached();
1818 }
1819
1820 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1821 get_field(s->fields, v3), fn);
1822 return DISAS_NEXT;
1823 }
1824
1825 static DisasJumpType op_vnn(DisasContext *s, DisasOps *o)
1826 {
1827 gen_gvec_fn_3(nand, ES_8, get_field(s->fields, v1),
1828 get_field(s->fields, v2), get_field(s->fields, v3));
1829 return DISAS_NEXT;
1830 }
1831
1832 static DisasJumpType op_vno(DisasContext *s, DisasOps *o)
1833 {
1834 gen_gvec_fn_3(nor, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1835 get_field(s->fields, v3));
1836 return DISAS_NEXT;
1837 }
1838
1839 static DisasJumpType op_vnx(DisasContext *s, DisasOps *o)
1840 {
1841 gen_gvec_fn_3(eqv, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1842 get_field(s->fields, v3));
1843 return DISAS_NEXT;
1844 }
1845
1846 static DisasJumpType op_vo(DisasContext *s, DisasOps *o)
1847 {
1848 gen_gvec_fn_3(or, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1849 get_field(s->fields, v3));
1850 return DISAS_NEXT;
1851 }
1852
1853 static DisasJumpType op_voc(DisasContext *s, DisasOps *o)
1854 {
1855 gen_gvec_fn_3(orc, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1856 get_field(s->fields, v3));
1857 return DISAS_NEXT;
1858 }
1859
1860 static DisasJumpType op_vpopct(DisasContext *s, DisasOps *o)
1861 {
1862 const uint8_t es = get_field(s->fields, m3);
1863 static const GVecGen2 g[4] = {
1864 { .fno = gen_helper_gvec_vpopct8, },
1865 { .fno = gen_helper_gvec_vpopct16, },
1866 { .fni4 = tcg_gen_ctpop_i32, },
1867 { .fni8 = tcg_gen_ctpop_i64, },
1868 };
1869
1870 if (es > ES_64 || (es != ES_8 && !s390_has_feat(S390_FEAT_VECTOR_ENH))) {
1871 gen_program_exception(s, PGM_SPECIFICATION);
1872 return DISAS_NORETURN;
1873 }
1874
1875 gen_gvec_2(get_field(s->fields, v1), get_field(s->fields, v2), &g[es]);
1876 return DISAS_NEXT;
1877 }
1878
1879 static void gen_rll_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1880 {
1881 TCGv_i32 t0 = tcg_temp_new_i32();
1882
1883 tcg_gen_andi_i32(t0, b, 31);
1884 tcg_gen_rotl_i32(d, a, t0);
1885 tcg_temp_free_i32(t0);
1886 }
1887
1888 static void gen_rll_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1889 {
1890 TCGv_i64 t0 = tcg_temp_new_i64();
1891
1892 tcg_gen_andi_i64(t0, b, 63);
1893 tcg_gen_rotl_i64(d, a, t0);
1894 tcg_temp_free_i64(t0);
1895 }
1896
1897 static DisasJumpType op_verllv(DisasContext *s, DisasOps *o)
1898 {
1899 const uint8_t es = get_field(s->fields, m4);
1900 static const GVecGen3 g[4] = {
1901 { .fno = gen_helper_gvec_verllv8, },
1902 { .fno = gen_helper_gvec_verllv16, },
1903 { .fni4 = gen_rll_i32, },
1904 { .fni8 = gen_rll_i64, },
1905 };
1906
1907 if (es > ES_64) {
1908 gen_program_exception(s, PGM_SPECIFICATION);
1909 return DISAS_NORETURN;
1910 }
1911
1912 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1913 get_field(s->fields, v3), &g[es]);
1914 return DISAS_NEXT;
1915 }
1916
1917 static DisasJumpType op_verll(DisasContext *s, DisasOps *o)
1918 {
1919 const uint8_t es = get_field(s->fields, m4);
1920 static const GVecGen2s g[4] = {
1921 { .fno = gen_helper_gvec_verll8, },
1922 { .fno = gen_helper_gvec_verll16, },
1923 { .fni4 = gen_rll_i32, },
1924 { .fni8 = gen_rll_i64, },
1925 };
1926
1927 if (es > ES_64) {
1928 gen_program_exception(s, PGM_SPECIFICATION);
1929 return DISAS_NORETURN;
1930 }
1931 gen_gvec_2s(get_field(s->fields, v1), get_field(s->fields, v3), o->addr1,
1932 &g[es]);
1933 return DISAS_NEXT;
1934 }
1935
1936 static void gen_rim_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, int32_t c)
1937 {
1938 TCGv_i32 t = tcg_temp_new_i32();
1939
1940 tcg_gen_rotli_i32(t, a, c & 31);
1941 tcg_gen_and_i32(t, t, b);
1942 tcg_gen_andc_i32(d, d, b);
1943 tcg_gen_or_i32(d, d, t);
1944
1945 tcg_temp_free_i32(t);
1946 }
1947
1948 static void gen_rim_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, int64_t c)
1949 {
1950 TCGv_i64 t = tcg_temp_new_i64();
1951
1952 tcg_gen_rotli_i64(t, a, c & 63);
1953 tcg_gen_and_i64(t, t, b);
1954 tcg_gen_andc_i64(d, d, b);
1955 tcg_gen_or_i64(d, d, t);
1956
1957 tcg_temp_free_i64(t);
1958 }
1959
1960 static DisasJumpType op_verim(DisasContext *s, DisasOps *o)
1961 {
1962 const uint8_t es = get_field(s->fields, m5);
1963 const uint8_t i4 = get_field(s->fields, i4) &
1964 (NUM_VEC_ELEMENT_BITS(es) - 1);
1965 static const GVecGen3i g[4] = {
1966 { .fno = gen_helper_gvec_verim8, },
1967 { .fno = gen_helper_gvec_verim16, },
1968 { .fni4 = gen_rim_i32,
1969 .load_dest = true, },
1970 { .fni8 = gen_rim_i64,
1971 .load_dest = true, },
1972 };
1973
1974 if (es > ES_64) {
1975 gen_program_exception(s, PGM_SPECIFICATION);
1976 return DISAS_NORETURN;
1977 }
1978
1979 gen_gvec_3i(get_field(s->fields, v1), get_field(s->fields, v2),
1980 get_field(s->fields, v3), i4, &g[es]);
1981 return DISAS_NEXT;
1982 }
1983
1984 static DisasJumpType op_vesv(DisasContext *s, DisasOps *o)
1985 {
1986 const uint8_t es = get_field(s->fields, m4);
1987 const uint8_t v1 = get_field(s->fields, v1);
1988 const uint8_t v2 = get_field(s->fields, v2);
1989 const uint8_t v3 = get_field(s->fields, v3);
1990
1991 if (es > ES_64) {
1992 gen_program_exception(s, PGM_SPECIFICATION);
1993 return DISAS_NORETURN;
1994 }
1995
1996 switch (s->fields->op2) {
1997 case 0x70:
1998 gen_gvec_fn_3(shlv, es, v1, v2, v3);
1999 break;
2000 case 0x7a:
2001 gen_gvec_fn_3(sarv, es, v1, v2, v3);
2002 break;
2003 case 0x78:
2004 gen_gvec_fn_3(shrv, es, v1, v2, v3);
2005 break;
2006 default:
2007 g_assert_not_reached();
2008 }
2009 return DISAS_NEXT;
2010 }
2011
2012 static DisasJumpType op_ves(DisasContext *s, DisasOps *o)
2013 {
2014 const uint8_t es = get_field(s->fields, m4);
2015 const uint8_t d2 = get_field(s->fields, d2) &
2016 (NUM_VEC_ELEMENT_BITS(es) - 1);
2017 const uint8_t v1 = get_field(s->fields, v1);
2018 const uint8_t v3 = get_field(s->fields, v3);
2019 TCGv_i32 shift;
2020
2021 if (es > ES_64) {
2022 gen_program_exception(s, PGM_SPECIFICATION);
2023 return DISAS_NORETURN;
2024 }
2025
2026 if (likely(!get_field(s->fields, b2))) {
2027 switch (s->fields->op2) {
2028 case 0x30:
2029 gen_gvec_fn_2i(shli, es, v1, v3, d2);
2030 break;
2031 case 0x3a:
2032 gen_gvec_fn_2i(sari, es, v1, v3, d2);
2033 break;
2034 case 0x38:
2035 gen_gvec_fn_2i(shri, es, v1, v3, d2);
2036 break;
2037 default:
2038 g_assert_not_reached();
2039 }
2040 } else {
2041 shift = tcg_temp_new_i32();
2042 tcg_gen_extrl_i64_i32(shift, o->addr1);
2043 tcg_gen_andi_i32(shift, shift, NUM_VEC_ELEMENT_BITS(es) - 1);
2044 switch (s->fields->op2) {
2045 case 0x30:
2046 gen_gvec_fn_2s(shls, es, v1, v3, shift);
2047 break;
2048 case 0x3a:
2049 gen_gvec_fn_2s(sars, es, v1, v3, shift);
2050 break;
2051 case 0x38:
2052 gen_gvec_fn_2s(shrs, es, v1, v3, shift);
2053 break;
2054 default:
2055 g_assert_not_reached();
2056 }
2057 tcg_temp_free_i32(shift);
2058 }
2059 return DISAS_NEXT;
2060 }
2061
2062 static DisasJumpType op_vsl(DisasContext *s, DisasOps *o)
2063 {
2064 TCGv_i64 shift = tcg_temp_new_i64();
2065
2066 read_vec_element_i64(shift, get_field(s->fields, v3), 7, ES_8);
2067 if (s->fields->op2 == 0x74) {
2068 tcg_gen_andi_i64(shift, shift, 0x7);
2069 } else {
2070 tcg_gen_andi_i64(shift, shift, 0x78);
2071 }
2072
2073 gen_gvec_2i_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2074 shift, 0, gen_helper_gvec_vsl);
2075 tcg_temp_free_i64(shift);
2076 return DISAS_NEXT;
2077 }
2078
2079 static DisasJumpType op_vsldb(DisasContext *s, DisasOps *o)
2080 {
2081 const uint8_t i4 = get_field(s->fields, i4) & 0xf;
2082 const int left_shift = (i4 & 7) * 8;
2083 const int right_shift = 64 - left_shift;
2084 TCGv_i64 t0 = tcg_temp_new_i64();
2085 TCGv_i64 t1 = tcg_temp_new_i64();
2086 TCGv_i64 t2 = tcg_temp_new_i64();
2087
2088 if ((i4 & 8) == 0) {
2089 read_vec_element_i64(t0, get_field(s->fields, v2), 0, ES_64);
2090 read_vec_element_i64(t1, get_field(s->fields, v2), 1, ES_64);
2091 read_vec_element_i64(t2, get_field(s->fields, v3), 0, ES_64);
2092 } else {
2093 read_vec_element_i64(t0, get_field(s->fields, v2), 1, ES_64);
2094 read_vec_element_i64(t1, get_field(s->fields, v3), 0, ES_64);
2095 read_vec_element_i64(t2, get_field(s->fields, v3), 1, ES_64);
2096 }
2097 tcg_gen_extract2_i64(t0, t1, t0, right_shift);
2098 tcg_gen_extract2_i64(t1, t2, t1, right_shift);
2099 write_vec_element_i64(t0, get_field(s->fields, v1), 0, ES_64);
2100 write_vec_element_i64(t1, get_field(s->fields, v1), 1, ES_64);
2101
2102 tcg_temp_free(t0);
2103 tcg_temp_free(t1);
2104 tcg_temp_free(t2);
2105 return DISAS_NEXT;
2106 }
2107
2108 static DisasJumpType op_vsra(DisasContext *s, DisasOps *o)
2109 {
2110 TCGv_i64 shift = tcg_temp_new_i64();
2111
2112 read_vec_element_i64(shift, get_field(s->fields, v3), 7, ES_8);
2113 if (s->fields->op2 == 0x7e) {
2114 tcg_gen_andi_i64(shift, shift, 0x7);
2115 } else {
2116 tcg_gen_andi_i64(shift, shift, 0x78);
2117 }
2118
2119 gen_gvec_2i_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2120 shift, 0, gen_helper_gvec_vsra);
2121 tcg_temp_free_i64(shift);
2122 return DISAS_NEXT;
2123 }
2124
2125 static DisasJumpType op_vsrl(DisasContext *s, DisasOps *o)
2126 {
2127 TCGv_i64 shift = tcg_temp_new_i64();
2128
2129 read_vec_element_i64(shift, get_field(s->fields, v3), 7, ES_8);
2130 if (s->fields->op2 == 0x7c) {
2131 tcg_gen_andi_i64(shift, shift, 0x7);
2132 } else {
2133 tcg_gen_andi_i64(shift, shift, 0x78);
2134 }
2135
2136 gen_gvec_2i_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2137 shift, 0, gen_helper_gvec_vsrl);
2138 tcg_temp_free_i64(shift);
2139 return DISAS_NEXT;
2140 }
2141
2142 static DisasJumpType op_vs(DisasContext *s, DisasOps *o)
2143 {
2144 const uint8_t es = get_field(s->fields, m4);
2145
2146 if (es > ES_128) {
2147 gen_program_exception(s, PGM_SPECIFICATION);
2148 return DISAS_NORETURN;
2149 } else if (es == ES_128) {
2150 gen_gvec128_3_i64(tcg_gen_sub2_i64, get_field(s->fields, v1),
2151 get_field(s->fields, v2), get_field(s->fields, v3));
2152 return DISAS_NEXT;
2153 }
2154 gen_gvec_fn_3(sub, es, get_field(s->fields, v1), get_field(s->fields, v2),
2155 get_field(s->fields, v3));
2156 return DISAS_NEXT;
2157 }
2158
2159 static void gen_scbi_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2160 {
2161 tcg_gen_setcond_i32(TCG_COND_LTU, d, a, b);
2162 }
2163
2164 static void gen_scbi_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2165 {
2166 tcg_gen_setcond_i64(TCG_COND_LTU, d, a, b);
2167 }
2168
2169 static void gen_scbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
2170 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2171 {
2172 TCGv_i64 th = tcg_temp_new_i64();
2173 TCGv_i64 tl = tcg_temp_new_i64();
2174 TCGv_i64 zero = tcg_const_i64(0);
2175
2176 tcg_gen_sub2_i64(tl, th, al, zero, bl, zero);
2177 tcg_gen_andi_i64(th, th, 1);
2178 tcg_gen_sub2_i64(tl, th, ah, zero, th, zero);
2179 tcg_gen_sub2_i64(tl, th, tl, th, bh, zero);
2180 tcg_gen_andi_i64(dl, th, 1);
2181 tcg_gen_mov_i64(dh, zero);
2182
2183 tcg_temp_free_i64(th);
2184 tcg_temp_free_i64(tl);
2185 tcg_temp_free_i64(zero);
2186 }
2187
2188 static DisasJumpType op_vscbi(DisasContext *s, DisasOps *o)
2189 {
2190 const uint8_t es = get_field(s->fields, m4);
2191 static const GVecGen3 g[4] = {
2192 { .fno = gen_helper_gvec_vscbi8, },
2193 { .fno = gen_helper_gvec_vscbi16, },
2194 { .fni4 = gen_scbi_i32, },
2195 { .fni8 = gen_scbi_i64, },
2196 };
2197
2198 if (es > ES_128) {
2199 gen_program_exception(s, PGM_SPECIFICATION);
2200 return DISAS_NORETURN;
2201 } else if (es == ES_128) {
2202 gen_gvec128_3_i64(gen_scbi2_i64, get_field(s->fields, v1),
2203 get_field(s->fields, v2), get_field(s->fields, v3));
2204 return DISAS_NEXT;
2205 }
2206 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
2207 get_field(s->fields, v3), &g[es]);
2208 return DISAS_NEXT;
2209 }
2210
2211 static void gen_sbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
2212 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
2213 {
2214 TCGv_i64 tl = tcg_temp_new_i64();
2215 TCGv_i64 zero = tcg_const_i64(0);
2216
2217 tcg_gen_andi_i64(tl, cl, 1);
2218 tcg_gen_sub2_i64(dl, dh, al, ah, bl, bh);
2219 tcg_gen_sub2_i64(dl, dh, dl, dh, tl, zero);
2220 tcg_temp_free_i64(tl);
2221 tcg_temp_free_i64(zero);
2222 }
2223
2224 static DisasJumpType op_vsbi(DisasContext *s, DisasOps *o)
2225 {
2226 if (get_field(s->fields, m5) != ES_128) {
2227 gen_program_exception(s, PGM_SPECIFICATION);
2228 return DISAS_NORETURN;
2229 }
2230
2231 gen_gvec128_4_i64(gen_sbi2_i64, get_field(s->fields, v1),
2232 get_field(s->fields, v2), get_field(s->fields, v3),
2233 get_field(s->fields, v4));
2234 return DISAS_NEXT;
2235 }
2236
2237 static void gen_sbcbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
2238 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
2239 {
2240 TCGv_i64 th = tcg_temp_new_i64();
2241 TCGv_i64 tl = tcg_temp_new_i64();
2242 TCGv_i64 zero = tcg_const_i64(0);
2243
2244 tcg_gen_andi_i64(tl, cl, 1);
2245 tcg_gen_sub2_i64(tl, th, al, zero, tl, zero);
2246 tcg_gen_sub2_i64(tl, th, tl, th, bl, zero);
2247 tcg_gen_andi_i64(th, th, 1);
2248 tcg_gen_sub2_i64(tl, th, ah, zero, th, zero);
2249 tcg_gen_sub2_i64(tl, th, tl, th, bh, zero);
2250 tcg_gen_andi_i64(dl, th, 1);
2251 tcg_gen_mov_i64(dh, zero);
2252
2253 tcg_temp_free_i64(tl);
2254 tcg_temp_free_i64(th);
2255 tcg_temp_free_i64(zero);
2256 }
2257
2258 static DisasJumpType op_vsbcbi(DisasContext *s, DisasOps *o)
2259 {
2260 if (get_field(s->fields, m5) != ES_128) {
2261 gen_program_exception(s, PGM_SPECIFICATION);
2262 return DISAS_NORETURN;
2263 }
2264
2265 gen_gvec128_4_i64(gen_sbcbi2_i64, get_field(s->fields, v1),
2266 get_field(s->fields, v2), get_field(s->fields, v3),
2267 get_field(s->fields, v4));
2268 return DISAS_NEXT;
2269 }
2270
2271 static DisasJumpType op_vsumg(DisasContext *s, DisasOps *o)
2272 {
2273 const uint8_t es = get_field(s->fields, m4);
2274 TCGv_i64 sum, tmp;
2275 uint8_t dst_idx;
2276
2277 if (es == ES_8 || es > ES_32) {
2278 gen_program_exception(s, PGM_SPECIFICATION);
2279 return DISAS_NORETURN;
2280 }
2281
2282 sum = tcg_temp_new_i64();
2283 tmp = tcg_temp_new_i64();
2284 for (dst_idx = 0; dst_idx < 2; dst_idx++) {
2285 uint8_t idx = dst_idx * NUM_VEC_ELEMENTS(es) / 2;
2286 const uint8_t max_idx = idx + NUM_VEC_ELEMENTS(es) / 2 - 1;
2287
2288 read_vec_element_i64(sum, get_field(s->fields, v3), max_idx, es);
2289 for (; idx <= max_idx; idx++) {
2290 read_vec_element_i64(tmp, get_field(s->fields, v2), idx, es);
2291 tcg_gen_add_i64(sum, sum, tmp);
2292 }
2293 write_vec_element_i64(sum, get_field(s->fields, v1), dst_idx, ES_64);
2294 }
2295 tcg_temp_free_i64(sum);
2296 tcg_temp_free_i64(tmp);
2297 return DISAS_NEXT;
2298 }
2299
2300 static DisasJumpType op_vsumq(DisasContext *s, DisasOps *o)
2301 {
2302 const uint8_t es = get_field(s->fields, m4);
2303 const uint8_t max_idx = NUM_VEC_ELEMENTS(es) - 1;
2304 TCGv_i64 sumh, suml, zero, tmpl;
2305 uint8_t idx;
2306
2307 if (es < ES_32 || es > ES_64) {
2308 gen_program_exception(s, PGM_SPECIFICATION);
2309 return DISAS_NORETURN;
2310 }
2311
2312 sumh = tcg_const_i64(0);
2313 suml = tcg_temp_new_i64();
2314 zero = tcg_const_i64(0);
2315 tmpl = tcg_temp_new_i64();
2316
2317 read_vec_element_i64(suml, get_field(s->fields, v3), max_idx, es);
2318 for (idx = 0; idx <= max_idx; idx++) {
2319 read_vec_element_i64(tmpl, get_field(s->fields, v2), idx, es);
2320 tcg_gen_add2_i64(suml, sumh, suml, sumh, tmpl, zero);
2321 }
2322 write_vec_element_i64(sumh, get_field(s->fields, v1), 0, ES_64);
2323 write_vec_element_i64(suml, get_field(s->fields, v1), 1, ES_64);
2324
2325 tcg_temp_free_i64(sumh);
2326 tcg_temp_free_i64(suml);
2327 tcg_temp_free_i64(zero);
2328 tcg_temp_free_i64(tmpl);
2329 return DISAS_NEXT;
2330 }
2331
2332 static DisasJumpType op_vsum(DisasContext *s, DisasOps *o)
2333 {
2334 const uint8_t es = get_field(s->fields, m4);
2335 TCGv_i32 sum, tmp;
2336 uint8_t dst_idx;
2337
2338 if (es > ES_16) {
2339 gen_program_exception(s, PGM_SPECIFICATION);
2340 return DISAS_NORETURN;
2341 }
2342
2343 sum = tcg_temp_new_i32();
2344 tmp = tcg_temp_new_i32();
2345 for (dst_idx = 0; dst_idx < 4; dst_idx++) {
2346 uint8_t idx = dst_idx * NUM_VEC_ELEMENTS(es) / 4;
2347 const uint8_t max_idx = idx + NUM_VEC_ELEMENTS(es) / 4 - 1;
2348
2349 read_vec_element_i32(sum, get_field(s->fields, v3), max_idx, es);
2350 for (; idx <= max_idx; idx++) {
2351 read_vec_element_i32(tmp, get_field(s->fields, v2), idx, es);
2352 tcg_gen_add_i32(sum, sum, tmp);
2353 }
2354 write_vec_element_i32(sum, get_field(s->fields, v1), dst_idx, ES_32);
2355 }
2356 tcg_temp_free_i32(sum);
2357 tcg_temp_free_i32(tmp);
2358 return DISAS_NEXT;
2359 }
2360
2361 static DisasJumpType op_vtm(DisasContext *s, DisasOps *o)
2362 {
2363 gen_gvec_2_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2364 cpu_env, 0, gen_helper_gvec_vtm);
2365 set_cc_static(s);
2366 return DISAS_NEXT;
2367 }
2368
2369 static DisasJumpType op_vfae(DisasContext *s, DisasOps *o)
2370 {
2371 const uint8_t es = get_field(s->fields, m4);
2372 const uint8_t m5 = get_field(s->fields, m5);
2373 static gen_helper_gvec_3 * const g[3] = {
2374 gen_helper_gvec_vfae8,
2375 gen_helper_gvec_vfae16,
2376 gen_helper_gvec_vfae32,
2377 };
2378 static gen_helper_gvec_3_ptr * const g_cc[3] = {
2379 gen_helper_gvec_vfae_cc8,
2380 gen_helper_gvec_vfae_cc16,
2381 gen_helper_gvec_vfae_cc32,
2382 };
2383 if (es > ES_32) {
2384 gen_program_exception(s, PGM_SPECIFICATION);
2385 return DISAS_NORETURN;
2386 }
2387
2388 if (extract32(m5, 0, 1)) {
2389 gen_gvec_3_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2390 get_field(s->fields, v3), cpu_env, m5, g_cc[es]);
2391 set_cc_static(s);
2392 } else {
2393 gen_gvec_3_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2394 get_field(s->fields, v3), m5, g[es]);
2395 }
2396 return DISAS_NEXT;
2397 }
2398
2399 static DisasJumpType op_vfee(DisasContext *s, DisasOps *o)
2400 {
2401 const uint8_t es = get_field(s->fields, m4);
2402 const uint8_t m5 = get_field(s->fields, m5);
2403 static gen_helper_gvec_3 * const g[3] = {
2404 gen_helper_gvec_vfee8,
2405 gen_helper_gvec_vfee16,
2406 gen_helper_gvec_vfee32,
2407 };
2408 static gen_helper_gvec_3_ptr * const g_cc[3] = {
2409 gen_helper_gvec_vfee_cc8,
2410 gen_helper_gvec_vfee_cc16,
2411 gen_helper_gvec_vfee_cc32,
2412 };
2413
2414 if (es > ES_32 || m5 & ~0x3) {
2415 gen_program_exception(s, PGM_SPECIFICATION);
2416 return DISAS_NORETURN;
2417 }
2418
2419 if (extract32(m5, 0, 1)) {
2420 gen_gvec_3_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2421 get_field(s->fields, v3), cpu_env, m5, g_cc[es]);
2422 set_cc_static(s);
2423 } else {
2424 gen_gvec_3_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2425 get_field(s->fields, v3), m5, g[es]);
2426 }
2427 return DISAS_NEXT;
2428 }
2429
2430 static DisasJumpType op_vfene(DisasContext *s, DisasOps *o)
2431 {
2432 const uint8_t es = get_field(s->fields, m4);
2433 const uint8_t m5 = get_field(s->fields, m5);
2434 static gen_helper_gvec_3 * const g[3] = {
2435 gen_helper_gvec_vfene8,
2436 gen_helper_gvec_vfene16,
2437 gen_helper_gvec_vfene32,
2438 };
2439 static gen_helper_gvec_3_ptr * const g_cc[3] = {
2440 gen_helper_gvec_vfene_cc8,
2441 gen_helper_gvec_vfene_cc16,
2442 gen_helper_gvec_vfene_cc32,
2443 };
2444
2445 if (es > ES_32 || m5 & ~0x3) {
2446 gen_program_exception(s, PGM_SPECIFICATION);
2447 return DISAS_NORETURN;
2448 }
2449
2450 if (extract32(m5, 0, 1)) {
2451 gen_gvec_3_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2452 get_field(s->fields, v3), cpu_env, m5, g_cc[es]);
2453 set_cc_static(s);
2454 } else {
2455 gen_gvec_3_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2456 get_field(s->fields, v3), m5, g[es]);
2457 }
2458 return DISAS_NEXT;
2459 }
2460
2461 static DisasJumpType op_vistr(DisasContext *s, DisasOps *o)
2462 {
2463 const uint8_t es = get_field(s->fields, m4);
2464 const uint8_t m5 = get_field(s->fields, m5);
2465 static gen_helper_gvec_2 * const g[3] = {
2466 gen_helper_gvec_vistr8,
2467 gen_helper_gvec_vistr16,
2468 gen_helper_gvec_vistr32,
2469 };
2470 static gen_helper_gvec_2_ptr * const g_cc[3] = {
2471 gen_helper_gvec_vistr_cc8,
2472 gen_helper_gvec_vistr_cc16,
2473 gen_helper_gvec_vistr_cc32,
2474 };
2475
2476 if (es > ES_32 || m5 & ~0x1) {
2477 gen_program_exception(s, PGM_SPECIFICATION);
2478 return DISAS_NORETURN;
2479 }
2480
2481 if (extract32(m5, 0, 1)) {
2482 gen_gvec_2_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2483 cpu_env, 0, g_cc[es]);
2484 set_cc_static(s);
2485 } else {
2486 gen_gvec_2_ool(get_field(s->fields, v1), get_field(s->fields, v2), 0,
2487 g[es]);
2488 }
2489 return DISAS_NEXT;
2490 }
2491
2492 static DisasJumpType op_vstrc(DisasContext *s, DisasOps *o)
2493 {
2494 const uint8_t es = get_field(s->fields, m5);
2495 const uint8_t m6 = get_field(s->fields, m6);
2496 static gen_helper_gvec_4 * const g[3] = {
2497 gen_helper_gvec_vstrc8,
2498 gen_helper_gvec_vstrc16,
2499 gen_helper_gvec_vstrc32,
2500 };
2501 static gen_helper_gvec_4 * const g_rt[3] = {
2502 gen_helper_gvec_vstrc_rt8,
2503 gen_helper_gvec_vstrc_rt16,
2504 gen_helper_gvec_vstrc_rt32,
2505 };
2506 static gen_helper_gvec_4_ptr * const g_cc[3] = {
2507 gen_helper_gvec_vstrc_cc8,
2508 gen_helper_gvec_vstrc_cc16,
2509 gen_helper_gvec_vstrc_cc32,
2510 };
2511 static gen_helper_gvec_4_ptr * const g_cc_rt[3] = {
2512 gen_helper_gvec_vstrc_cc_rt8,
2513 gen_helper_gvec_vstrc_cc_rt16,
2514 gen_helper_gvec_vstrc_cc_rt32,
2515 };
2516
2517 if (es > ES_32) {
2518 gen_program_exception(s, PGM_SPECIFICATION);
2519 return DISAS_NORETURN;
2520 }
2521
2522 if (extract32(m6, 0, 1)) {
2523 if (extract32(m6, 2, 1)) {
2524 gen_gvec_4_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2525 get_field(s->fields, v3), get_field(s->fields, v4),
2526 cpu_env, m6, g_cc_rt[es]);
2527 } else {
2528 gen_gvec_4_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2529 get_field(s->fields, v3), get_field(s->fields, v4),
2530 cpu_env, m6, g_cc[es]);
2531 }
2532 set_cc_static(s);
2533 } else {
2534 if (extract32(m6, 2, 1)) {
2535 gen_gvec_4_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2536 get_field(s->fields, v3), get_field(s->fields, v4),
2537 m6, g_rt[es]);
2538 } else {
2539 gen_gvec_4_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2540 get_field(s->fields, v3), get_field(s->fields, v4),
2541 m6, g[es]);
2542 }
2543 }
2544 return DISAS_NEXT;
2545 }
2546
2547 static DisasJumpType op_vfa(DisasContext *s, DisasOps *o)
2548 {
2549 const uint8_t fpf = get_field(s->fields, m4);
2550 const uint8_t m5 = get_field(s->fields, m5);
2551 const bool se = extract32(m5, 3, 1);
2552 gen_helper_gvec_3_ptr *fn;
2553
2554 if (fpf != FPF_LONG || extract32(m5, 0, 3)) {
2555 gen_program_exception(s, PGM_SPECIFICATION);
2556 return DISAS_NORETURN;
2557 }
2558
2559 switch (s->fields->op2) {
2560 case 0xe3:
2561 fn = se ? gen_helper_gvec_vfa64s : gen_helper_gvec_vfa64;
2562 break;
2563 default:
2564 g_assert_not_reached();
2565 }
2566 gen_gvec_3_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2567 get_field(s->fields, v3), cpu_env, 0, fn);
2568 return DISAS_NEXT;
2569 }
2570
2571 static DisasJumpType op_wfc(DisasContext *s, DisasOps *o)
2572 {
2573 const uint8_t fpf = get_field(s->fields, m3);
2574 const uint8_t m4 = get_field(s->fields, m4);
2575
2576 if (fpf != FPF_LONG || m4) {
2577 gen_program_exception(s, PGM_SPECIFICATION);
2578 return DISAS_NORETURN;
2579 }
2580
2581 if (s->fields->op2 == 0xcb) {
2582 gen_gvec_2_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2583 cpu_env, 0, gen_helper_gvec_wfc64);
2584 } else {
2585 gen_gvec_2_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2586 cpu_env, 0, gen_helper_gvec_wfk64);
2587 }
2588 set_cc_static(s);
2589 return DISAS_NEXT;
2590 }
2591
2592 static DisasJumpType op_vfc(DisasContext *s, DisasOps *o)
2593 {
2594 const uint8_t fpf = get_field(s->fields, m4);
2595 const uint8_t m5 = get_field(s->fields, m5);
2596 const uint8_t m6 = get_field(s->fields, m6);
2597 const bool se = extract32(m5, 3, 1);
2598 const bool cs = extract32(m6, 0, 1);
2599 gen_helper_gvec_3_ptr *fn;
2600
2601 if (fpf != FPF_LONG || extract32(m5, 0, 3) || extract32(m6, 1, 3)) {
2602 gen_program_exception(s, PGM_SPECIFICATION);
2603 return DISAS_NORETURN;
2604 }
2605
2606 if (cs) {
2607 switch (s->fields->op2) {
2608 case 0xe8:
2609 fn = se ? gen_helper_gvec_vfce64s_cc : gen_helper_gvec_vfce64_cc;
2610 break;
2611 case 0xeb:
2612 fn = se ? gen_helper_gvec_vfch64s_cc : gen_helper_gvec_vfch64_cc;
2613 break;
2614 case 0xea:
2615 fn = se ? gen_helper_gvec_vfche64s_cc : gen_helper_gvec_vfche64_cc;
2616 break;
2617 default:
2618 g_assert_not_reached();
2619 }
2620 } else {
2621 switch (s->fields->op2) {
2622 case 0xe8:
2623 fn = se ? gen_helper_gvec_vfce64s : gen_helper_gvec_vfce64;
2624 break;
2625 case 0xeb:
2626 fn = se ? gen_helper_gvec_vfch64s : gen_helper_gvec_vfch64;
2627 break;
2628 case 0xea:
2629 fn = se ? gen_helper_gvec_vfche64s : gen_helper_gvec_vfche64;
2630 break;
2631 default:
2632 g_assert_not_reached();
2633 }
2634 }
2635 gen_gvec_3_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2636 get_field(s->fields, v3), cpu_env, 0, fn);
2637 if (cs) {
2638 set_cc_static(s);
2639 }
2640 return DISAS_NEXT;
2641 }
AR7 emulation for QEMU