?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
s390x/tcg: Implement VECTOR FIND ANY ELEMENT EQUAL
[qemu/ar7.git] / target / s390x / translate_vx.inc.c
blobebd7a877f17dd5991d4d20d18895f86312669dd7
1 /*
2 * QEMU TCG support -- s390x vector instruction translation functions
3 *
4 * Copyright (C) 2019 Red Hat Inc
5 *
6 * Authors:
7 * David Hildenbrand <david@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13 /*
14 * For most instructions that use the same element size for reads and
15 * writes, we can use real gvec vector expansion, which potantially uses
16 * real host vector instructions. As they only work up to 64 bit elements,
17 * 128 bit elements (vector is a single element) have to be handled
18 * differently. Operations that are too complicated to encode via TCG ops
19 * are handled via gvec ool (out-of-line) handlers.
20 *
21 * As soon as instructions use different element sizes for reads and writes
22 * or access elements "out of their element scope" we expand them manually
23 * in fancy loops, as gvec expansion does not deal with actual element
24 * numbers and does also not support access to other elements.
25 *
26 * 128 bit elements:
27 * As we only have i32/i64, such elements have to be loaded into two
28 * i64 values and can then be processed e.g. by tcg_gen_add2_i64.
29 *
30 * Sizes:
31 * On s390x, the operand size (oprsz) and the maximum size (maxsz) are
32 * always 16 (128 bit). What gvec code calls "vece", s390x calls "es",
33 * a.k.a. "element size". These values nicely map to MO_8 ... MO_64. Only
34 * 128 bit element size has to be treated in a special way (MO_64 + 1).
35 * We will use ES_* instead of MO_* for this reason in this file.
36 *
37 * CC handling:
38 * As gvec ool-helpers can currently not return values (besides via
39 * pointers like vectors or cpu_env), whenever we have to set the CC and
40 * can't conclude the value from the result vector, we will directly
41 * set it in "env->cc_op" and mark it as static via set_cc_static()".
42 * Whenever this is done, the helper writes globals (cc_op).
43 */
44
45 #define NUM_VEC_ELEMENT_BYTES(es) (1 << (es))
46 #define NUM_VEC_ELEMENTS(es) (16 / NUM_VEC_ELEMENT_BYTES(es))
47 #define NUM_VEC_ELEMENT_BITS(es) (NUM_VEC_ELEMENT_BYTES(es) * BITS_PER_BYTE)
48
49 #define ES_8 MO_8
50 #define ES_16 MO_16
51 #define ES_32 MO_32
52 #define ES_64 MO_64
53 #define ES_128 4
54
55 static inline bool valid_vec_element(uint8_t enr, TCGMemOp es)
56 {
57 return !(enr & ~(NUM_VEC_ELEMENTS(es) - 1));
58 }
59
60 static void read_vec_element_i64(TCGv_i64 dst, uint8_t reg, uint8_t enr,
61 TCGMemOp memop)
62 {
63 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
64
65 switch (memop) {
66 case ES_8:
67 tcg_gen_ld8u_i64(dst, cpu_env, offs);
68 break;
69 case ES_16:
70 tcg_gen_ld16u_i64(dst, cpu_env, offs);
71 break;
72 case ES_32:
73 tcg_gen_ld32u_i64(dst, cpu_env, offs);
74 break;
75 case ES_8 | MO_SIGN:
76 tcg_gen_ld8s_i64(dst, cpu_env, offs);
77 break;
78 case ES_16 | MO_SIGN:
79 tcg_gen_ld16s_i64(dst, cpu_env, offs);
80 break;
81 case ES_32 | MO_SIGN:
82 tcg_gen_ld32s_i64(dst, cpu_env, offs);
83 break;
84 case ES_64:
85 case ES_64 | MO_SIGN:
86 tcg_gen_ld_i64(dst, cpu_env, offs);
87 break;
88 default:
89 g_assert_not_reached();
90 }
91 }
92
93 static void read_vec_element_i32(TCGv_i32 dst, uint8_t reg, uint8_t enr,
94 TCGMemOp memop)
95 {
96 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
97
98 switch (memop) {
99 case ES_8:
100 tcg_gen_ld8u_i32(dst, cpu_env, offs);
101 break;
102 case ES_16:
103 tcg_gen_ld16u_i32(dst, cpu_env, offs);
104 break;
105 case ES_8 | MO_SIGN:
106 tcg_gen_ld8s_i32(dst, cpu_env, offs);
107 break;
108 case ES_16 | MO_SIGN:
109 tcg_gen_ld16s_i32(dst, cpu_env, offs);
110 break;
111 case ES_32:
112 case ES_32 | MO_SIGN:
113 tcg_gen_ld_i32(dst, cpu_env, offs);
114 break;
115 default:
116 g_assert_not_reached();
117 }
118 }
119
120 static void write_vec_element_i64(TCGv_i64 src, int reg, uint8_t enr,
121 TCGMemOp memop)
122 {
123 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
124
125 switch (memop) {
126 case ES_8:
127 tcg_gen_st8_i64(src, cpu_env, offs);
128 break;
129 case ES_16:
130 tcg_gen_st16_i64(src, cpu_env, offs);
131 break;
132 case ES_32:
133 tcg_gen_st32_i64(src, cpu_env, offs);
134 break;
135 case ES_64:
136 tcg_gen_st_i64(src, cpu_env, offs);
137 break;
138 default:
139 g_assert_not_reached();
140 }
141 }
142
143 static void write_vec_element_i32(TCGv_i32 src, int reg, uint8_t enr,
144 TCGMemOp memop)
145 {
146 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
147
148 switch (memop) {
149 case ES_8:
150 tcg_gen_st8_i32(src, cpu_env, offs);
151 break;
152 case ES_16:
153 tcg_gen_st16_i32(src, cpu_env, offs);
154 break;
155 case ES_32:
156 tcg_gen_st_i32(src, cpu_env, offs);
157 break;
158 default:
159 g_assert_not_reached();
160 }
161 }
162
163 static void get_vec_element_ptr_i64(TCGv_ptr ptr, uint8_t reg, TCGv_i64 enr,
164 uint8_t es)
165 {
166 TCGv_i64 tmp = tcg_temp_new_i64();
167
168 /* mask off invalid parts from the element nr */
169 tcg_gen_andi_i64(tmp, enr, NUM_VEC_ELEMENTS(es) - 1);
170
171 /* convert it to an element offset relative to cpu_env (vec_reg_offset() */
172 tcg_gen_shli_i64(tmp, tmp, es);
173 #ifndef HOST_WORDS_BIGENDIAN
174 tcg_gen_xori_i64(tmp, tmp, 8 - NUM_VEC_ELEMENT_BYTES(es));
175 #endif
176 tcg_gen_addi_i64(tmp, tmp, vec_full_reg_offset(reg));
177
178 /* generate the final ptr by adding cpu_env */
179 tcg_gen_trunc_i64_ptr(ptr, tmp);
180 tcg_gen_add_ptr(ptr, ptr, cpu_env);
181
182 tcg_temp_free_i64(tmp);
183 }
184
185 #define gen_gvec_2(v1, v2, gen) \
186 tcg_gen_gvec_2(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
187 16, 16, gen)
188 #define gen_gvec_2s(v1, v2, c, gen) \
189 tcg_gen_gvec_2s(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
190 16, 16, c, gen)
191 #define gen_gvec_2i_ool(v1, v2, c, data, fn) \
192 tcg_gen_gvec_2i_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
193 c, 16, 16, data, fn)
194 #define gen_gvec_2_ptr(v1, v2, ptr, data, fn) \
195 tcg_gen_gvec_2_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
196 ptr, 16, 16, data, fn)
197 #define gen_gvec_3(v1, v2, v3, gen) \
198 tcg_gen_gvec_3(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
199 vec_full_reg_offset(v3), 16, 16, gen)
200 #define gen_gvec_3_ool(v1, v2, v3, data, fn) \
201 tcg_gen_gvec_3_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
202 vec_full_reg_offset(v3), 16, 16, data, fn)
203 #define gen_gvec_3_ptr(v1, v2, v3, ptr, data, fn) \
204 tcg_gen_gvec_3_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
205 vec_full_reg_offset(v3), ptr, 16, 16, data, fn)
206 #define gen_gvec_3i(v1, v2, v3, c, gen) \
207 tcg_gen_gvec_3i(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
208 vec_full_reg_offset(v3), c, 16, 16, gen)
209 #define gen_gvec_4(v1, v2, v3, v4, gen) \
210 tcg_gen_gvec_4(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
211 vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
212 16, 16, gen)
213 #define gen_gvec_4_ool(v1, v2, v3, v4, data, fn) \
214 tcg_gen_gvec_4_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
215 vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
216 16, 16, data, fn)
217 #define gen_gvec_dup_i64(es, v1, c) \
218 tcg_gen_gvec_dup_i64(es, vec_full_reg_offset(v1), 16, 16, c)
219 #define gen_gvec_mov(v1, v2) \
220 tcg_gen_gvec_mov(0, vec_full_reg_offset(v1), vec_full_reg_offset(v2), 16, \
221 16)
222 #define gen_gvec_dup64i(v1, c) \
223 tcg_gen_gvec_dup64i(vec_full_reg_offset(v1), 16, 16, c)
224 #define gen_gvec_fn_2(fn, es, v1, v2) \
225 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
226 16, 16)
227 #define gen_gvec_fn_2i(fn, es, v1, v2, c) \
228 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
229 c, 16, 16)
230 #define gen_gvec_fn_2s(fn, es, v1, v2, s) \
231 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
232 s, 16, 16)
233 #define gen_gvec_fn_3(fn, es, v1, v2, v3) \
234 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
235 vec_full_reg_offset(v3), 16, 16)
236
237 /*
238 * Helper to carry out a 128 bit vector computation using 2 i64 values per
239 * vector.
240 */
241 typedef void (*gen_gvec128_3_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
242 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh);
243 static void gen_gvec128_3_i64(gen_gvec128_3_i64_fn fn, uint8_t d, uint8_t a,
244 uint8_t b)
245 {
246 TCGv_i64 dh = tcg_temp_new_i64();
247 TCGv_i64 dl = tcg_temp_new_i64();
248 TCGv_i64 ah = tcg_temp_new_i64();
249 TCGv_i64 al = tcg_temp_new_i64();
250 TCGv_i64 bh = tcg_temp_new_i64();
251 TCGv_i64 bl = tcg_temp_new_i64();
252
253 read_vec_element_i64(ah, a, 0, ES_64);
254 read_vec_element_i64(al, a, 1, ES_64);
255 read_vec_element_i64(bh, b, 0, ES_64);
256 read_vec_element_i64(bl, b, 1, ES_64);
257 fn(dl, dh, al, ah, bl, bh);
258 write_vec_element_i64(dh, d, 0, ES_64);
259 write_vec_element_i64(dl, d, 1, ES_64);
260
261 tcg_temp_free_i64(dh);
262 tcg_temp_free_i64(dl);
263 tcg_temp_free_i64(ah);
264 tcg_temp_free_i64(al);
265 tcg_temp_free_i64(bh);
266 tcg_temp_free_i64(bl);
267 }
268
269 typedef void (*gen_gvec128_4_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
270 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh,
271 TCGv_i64 cl, TCGv_i64 ch);
272 static void gen_gvec128_4_i64(gen_gvec128_4_i64_fn fn, uint8_t d, uint8_t a,
273 uint8_t b, uint8_t c)
274 {
275 TCGv_i64 dh = tcg_temp_new_i64();
276 TCGv_i64 dl = tcg_temp_new_i64();
277 TCGv_i64 ah = tcg_temp_new_i64();
278 TCGv_i64 al = tcg_temp_new_i64();
279 TCGv_i64 bh = tcg_temp_new_i64();
280 TCGv_i64 bl = tcg_temp_new_i64();
281 TCGv_i64 ch = tcg_temp_new_i64();
282 TCGv_i64 cl = tcg_temp_new_i64();
283
284 read_vec_element_i64(ah, a, 0, ES_64);
285 read_vec_element_i64(al, a, 1, ES_64);
286 read_vec_element_i64(bh, b, 0, ES_64);
287 read_vec_element_i64(bl, b, 1, ES_64);
288 read_vec_element_i64(ch, c, 0, ES_64);
289 read_vec_element_i64(cl, c, 1, ES_64);
290 fn(dl, dh, al, ah, bl, bh, cl, ch);
291 write_vec_element_i64(dh, d, 0, ES_64);
292 write_vec_element_i64(dl, d, 1, ES_64);
293
294 tcg_temp_free_i64(dh);
295 tcg_temp_free_i64(dl);
296 tcg_temp_free_i64(ah);
297 tcg_temp_free_i64(al);
298 tcg_temp_free_i64(bh);
299 tcg_temp_free_i64(bl);
300 tcg_temp_free_i64(ch);
301 tcg_temp_free_i64(cl);
302 }
303
304 static void gen_gvec_dupi(uint8_t es, uint8_t reg, uint64_t c)
305 {
306 switch (es) {
307 case ES_8:
308 tcg_gen_gvec_dup8i(vec_full_reg_offset(reg), 16, 16, c);
309 break;
310 case ES_16:
311 tcg_gen_gvec_dup16i(vec_full_reg_offset(reg), 16, 16, c);
312 break;
313 case ES_32:
314 tcg_gen_gvec_dup32i(vec_full_reg_offset(reg), 16, 16, c);
315 break;
316 case ES_64:
317 gen_gvec_dup64i(reg, c);
318 break;
319 default:
320 g_assert_not_reached();
321 }
322 }
323
324 static void zero_vec(uint8_t reg)
325 {
326 tcg_gen_gvec_dup8i(vec_full_reg_offset(reg), 16, 16, 0);
327 }
328
329 static void gen_addi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
330 uint64_t b)
331 {
332 TCGv_i64 bl = tcg_const_i64(b);
333 TCGv_i64 bh = tcg_const_i64(0);
334
335 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
336 tcg_temp_free_i64(bl);
337 tcg_temp_free_i64(bh);
338 }
339
340 static DisasJumpType op_vge(DisasContext *s, DisasOps *o)
341 {
342 const uint8_t es = s->insn->data;
343 const uint8_t enr = get_field(s->fields, m3);
344 TCGv_i64 tmp;
345
346 if (!valid_vec_element(enr, es)) {
347 gen_program_exception(s, PGM_SPECIFICATION);
348 return DISAS_NORETURN;
349 }
350
351 tmp = tcg_temp_new_i64();
352 read_vec_element_i64(tmp, get_field(s->fields, v2), enr, es);
353 tcg_gen_add_i64(o->addr1, o->addr1, tmp);
354 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
355
356 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
357 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
358 tcg_temp_free_i64(tmp);
359 return DISAS_NEXT;
360 }
361
362 static uint64_t generate_byte_mask(uint8_t mask)
363 {
364 uint64_t r = 0;
365 int i;
366
367 for (i = 0; i < 8; i++) {
368 if ((mask >> i) & 1) {
369 r |= 0xffull << (i * 8);
370 }
371 }
372 return r;
373 }
374
375 static DisasJumpType op_vgbm(DisasContext *s, DisasOps *o)
376 {
377 const uint16_t i2 = get_field(s->fields, i2);
378
379 if (i2 == (i2 & 0xff) * 0x0101) {
380 /*
381 * Masks for both 64 bit elements of the vector are the same.
382 * Trust tcg to produce a good constant loading.
383 */
384 gen_gvec_dup64i(get_field(s->fields, v1),
385 generate_byte_mask(i2 & 0xff));
386 } else {
387 TCGv_i64 t = tcg_temp_new_i64();
388
389 tcg_gen_movi_i64(t, generate_byte_mask(i2 >> 8));
390 write_vec_element_i64(t, get_field(s->fields, v1), 0, ES_64);
391 tcg_gen_movi_i64(t, generate_byte_mask(i2));
392 write_vec_element_i64(t, get_field(s->fields, v1), 1, ES_64);
393 tcg_temp_free_i64(t);
394 }
395 return DISAS_NEXT;
396 }
397
398 static DisasJumpType op_vgm(DisasContext *s, DisasOps *o)
399 {
400 const uint8_t es = get_field(s->fields, m4);
401 const uint8_t bits = NUM_VEC_ELEMENT_BITS(es);
402 const uint8_t i2 = get_field(s->fields, i2) & (bits - 1);
403 const uint8_t i3 = get_field(s->fields, i3) & (bits - 1);
404 uint64_t mask = 0;
405 int i;
406
407 if (es > ES_64) {
408 gen_program_exception(s, PGM_SPECIFICATION);
409 return DISAS_NORETURN;
410 }
411
412 /* generate the mask - take care of wrapping */
413 for (i = i2; ; i = (i + 1) % bits) {
414 mask |= 1ull << (bits - i - 1);
415 if (i == i3) {
416 break;
417 }
418 }
419
420 gen_gvec_dupi(es, get_field(s->fields, v1), mask);
421 return DISAS_NEXT;
422 }
423
424 static DisasJumpType op_vl(DisasContext *s, DisasOps *o)
425 {
426 TCGv_i64 t0 = tcg_temp_new_i64();
427 TCGv_i64 t1 = tcg_temp_new_i64();
428
429 tcg_gen_qemu_ld_i64(t0, o->addr1, get_mem_index(s), MO_TEQ);
430 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
431 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
432 write_vec_element_i64(t0, get_field(s->fields, v1), 0, ES_64);
433 write_vec_element_i64(t1, get_field(s->fields, v1), 1, ES_64);
434 tcg_temp_free(t0);
435 tcg_temp_free(t1);
436 return DISAS_NEXT;
437 }
438
439 static DisasJumpType op_vlr(DisasContext *s, DisasOps *o)
440 {
441 gen_gvec_mov(get_field(s->fields, v1), get_field(s->fields, v2));
442 return DISAS_NEXT;
443 }
444
445 static DisasJumpType op_vlrep(DisasContext *s, DisasOps *o)
446 {
447 const uint8_t es = get_field(s->fields, m3);
448 TCGv_i64 tmp;
449
450 if (es > ES_64) {
451 gen_program_exception(s, PGM_SPECIFICATION);
452 return DISAS_NORETURN;
453 }
454
455 tmp = tcg_temp_new_i64();
456 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
457 gen_gvec_dup_i64(es, get_field(s->fields, v1), tmp);
458 tcg_temp_free_i64(tmp);
459 return DISAS_NEXT;
460 }
461
462 static DisasJumpType op_vle(DisasContext *s, DisasOps *o)
463 {
464 const uint8_t es = s->insn->data;
465 const uint8_t enr = get_field(s->fields, m3);
466 TCGv_i64 tmp;
467
468 if (!valid_vec_element(enr, es)) {
469 gen_program_exception(s, PGM_SPECIFICATION);
470 return DISAS_NORETURN;
471 }
472
473 tmp = tcg_temp_new_i64();
474 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
475 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
476 tcg_temp_free_i64(tmp);
477 return DISAS_NEXT;
478 }
479
480 static DisasJumpType op_vlei(DisasContext *s, DisasOps *o)
481 {
482 const uint8_t es = s->insn->data;
483 const uint8_t enr = get_field(s->fields, m3);
484 TCGv_i64 tmp;
485
486 if (!valid_vec_element(enr, es)) {
487 gen_program_exception(s, PGM_SPECIFICATION);
488 return DISAS_NORETURN;
489 }
490
491 tmp = tcg_const_i64((int16_t)get_field(s->fields, i2));
492 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
493 tcg_temp_free_i64(tmp);
494 return DISAS_NEXT;
495 }
496
497 static DisasJumpType op_vlgv(DisasContext *s, DisasOps *o)
498 {
499 const uint8_t es = get_field(s->fields, m4);
500 TCGv_ptr ptr;
501
502 if (es > ES_64) {
503 gen_program_exception(s, PGM_SPECIFICATION);
504 return DISAS_NORETURN;
505 }
506
507 /* fast path if we don't need the register content */
508 if (!get_field(s->fields, b2)) {
509 uint8_t enr = get_field(s->fields, d2) & (NUM_VEC_ELEMENTS(es) - 1);
510
511 read_vec_element_i64(o->out, get_field(s->fields, v3), enr, es);
512 return DISAS_NEXT;
513 }
514
515 ptr = tcg_temp_new_ptr();
516 get_vec_element_ptr_i64(ptr, get_field(s->fields, v3), o->addr1, es);
517 switch (es) {
518 case ES_8:
519 tcg_gen_ld8u_i64(o->out, ptr, 0);
520 break;
521 case ES_16:
522 tcg_gen_ld16u_i64(o->out, ptr, 0);
523 break;
524 case ES_32:
525 tcg_gen_ld32u_i64(o->out, ptr, 0);
526 break;
527 case ES_64:
528 tcg_gen_ld_i64(o->out, ptr, 0);
529 break;
530 default:
531 g_assert_not_reached();
532 }
533 tcg_temp_free_ptr(ptr);
534
535 return DISAS_NEXT;
536 }
537
538 static DisasJumpType op_vllez(DisasContext *s, DisasOps *o)
539 {
540 uint8_t es = get_field(s->fields, m3);
541 uint8_t enr;
542 TCGv_i64 t;
543
544 switch (es) {
545 /* rightmost sub-element of leftmost doubleword */
546 case ES_8:
547 enr = 7;
548 break;
549 case ES_16:
550 enr = 3;
551 break;
552 case ES_32:
553 enr = 1;
554 break;
555 case ES_64:
556 enr = 0;
557 break;
558 /* leftmost sub-element of leftmost doubleword */
559 case 6:
560 if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
561 es = ES_32;
562 enr = 0;
563 break;
564 }
565 default:
566 /* fallthrough */
567 gen_program_exception(s, PGM_SPECIFICATION);
568 return DISAS_NORETURN;
569 }
570
571 t = tcg_temp_new_i64();
572 tcg_gen_qemu_ld_i64(t, o->addr1, get_mem_index(s), MO_TE | es);
573 zero_vec(get_field(s->fields, v1));
574 write_vec_element_i64(t, get_field(s->fields, v1), enr, es);
575 tcg_temp_free_i64(t);
576 return DISAS_NEXT;
577 }
578
579 static DisasJumpType op_vlm(DisasContext *s, DisasOps *o)
580 {
581 const uint8_t v3 = get_field(s->fields, v3);
582 uint8_t v1 = get_field(s->fields, v1);
583 TCGv_i64 t0, t1;
584
585 if (v3 < v1 || (v3 - v1 + 1) > 16) {
586 gen_program_exception(s, PGM_SPECIFICATION);
587 return DISAS_NORETURN;
588 }
589
590 /*
591 * Check for possible access exceptions by trying to load the last
592 * element. The first element will be checked first next.
593 */
594 t0 = tcg_temp_new_i64();
595 t1 = tcg_temp_new_i64();
596 gen_addi_and_wrap_i64(s, t0, o->addr1, (v3 - v1) * 16 + 8);
597 tcg_gen_qemu_ld_i64(t0, t0, get_mem_index(s), MO_TEQ);
598
599 for (;; v1++) {
600 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
601 write_vec_element_i64(t1, v1, 0, ES_64);
602 if (v1 == v3) {
603 break;
604 }
605 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
606 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
607 write_vec_element_i64(t1, v1, 1, ES_64);
608 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
609 }
610
611 /* Store the last element, loaded first */
612 write_vec_element_i64(t0, v1, 1, ES_64);
613
614 tcg_temp_free_i64(t0);
615 tcg_temp_free_i64(t1);
616 return DISAS_NEXT;
617 }
618
619 static DisasJumpType op_vlbb(DisasContext *s, DisasOps *o)
620 {
621 const int64_t block_size = (1ull << (get_field(s->fields, m3) + 6));
622 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
623 TCGv_ptr a0;
624 TCGv_i64 bytes;
625
626 if (get_field(s->fields, m3) > 6) {
627 gen_program_exception(s, PGM_SPECIFICATION);
628 return DISAS_NORETURN;
629 }
630
631 bytes = tcg_temp_new_i64();
632 a0 = tcg_temp_new_ptr();
633 /* calculate the number of bytes until the next block boundary */
634 tcg_gen_ori_i64(bytes, o->addr1, -block_size);
635 tcg_gen_neg_i64(bytes, bytes);
636
637 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
638 gen_helper_vll(cpu_env, a0, o->addr1, bytes);
639 tcg_temp_free_i64(bytes);
640 tcg_temp_free_ptr(a0);
641 return DISAS_NEXT;
642 }
643
644 static DisasJumpType op_vlvg(DisasContext *s, DisasOps *o)
645 {
646 const uint8_t es = get_field(s->fields, m4);
647 TCGv_ptr ptr;
648
649 if (es > ES_64) {
650 gen_program_exception(s, PGM_SPECIFICATION);
651 return DISAS_NORETURN;
652 }
653
654 /* fast path if we don't need the register content */
655 if (!get_field(s->fields, b2)) {
656 uint8_t enr = get_field(s->fields, d2) & (NUM_VEC_ELEMENTS(es) - 1);
657
658 write_vec_element_i64(o->in2, get_field(s->fields, v1), enr, es);
659 return DISAS_NEXT;
660 }
661
662 ptr = tcg_temp_new_ptr();
663 get_vec_element_ptr_i64(ptr, get_field(s->fields, v1), o->addr1, es);
664 switch (es) {
665 case ES_8:
666 tcg_gen_st8_i64(o->in2, ptr, 0);
667 break;
668 case ES_16:
669 tcg_gen_st16_i64(o->in2, ptr, 0);
670 break;
671 case ES_32:
672 tcg_gen_st32_i64(o->in2, ptr, 0);
673 break;
674 case ES_64:
675 tcg_gen_st_i64(o->in2, ptr, 0);
676 break;
677 default:
678 g_assert_not_reached();
679 }
680 tcg_temp_free_ptr(ptr);
681
682 return DISAS_NEXT;
683 }
684
685 static DisasJumpType op_vlvgp(DisasContext *s, DisasOps *o)
686 {
687 write_vec_element_i64(o->in1, get_field(s->fields, v1), 0, ES_64);
688 write_vec_element_i64(o->in2, get_field(s->fields, v1), 1, ES_64);
689 return DISAS_NEXT;
690 }
691
692 static DisasJumpType op_vll(DisasContext *s, DisasOps *o)
693 {
694 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
695 TCGv_ptr a0 = tcg_temp_new_ptr();
696
697 /* convert highest index into an actual length */
698 tcg_gen_addi_i64(o->in2, o->in2, 1);
699 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
700 gen_helper_vll(cpu_env, a0, o->addr1, o->in2);
701 tcg_temp_free_ptr(a0);
702 return DISAS_NEXT;
703 }
704
705 static DisasJumpType op_vmr(DisasContext *s, DisasOps *o)
706 {
707 const uint8_t v1 = get_field(s->fields, v1);
708 const uint8_t v2 = get_field(s->fields, v2);
709 const uint8_t v3 = get_field(s->fields, v3);
710 const uint8_t es = get_field(s->fields, m4);
711 int dst_idx, src_idx;
712 TCGv_i64 tmp;
713
714 if (es > ES_64) {
715 gen_program_exception(s, PGM_SPECIFICATION);
716 return DISAS_NORETURN;
717 }
718
719 tmp = tcg_temp_new_i64();
720 if (s->fields->op2 == 0x61) {
721 /* iterate backwards to avoid overwriting data we might need later */
722 for (dst_idx = NUM_VEC_ELEMENTS(es) - 1; dst_idx >= 0; dst_idx--) {
723 src_idx = dst_idx / 2;
724 if (dst_idx % 2 == 0) {
725 read_vec_element_i64(tmp, v2, src_idx, es);
726 } else {
727 read_vec_element_i64(tmp, v3, src_idx, es);
728 }
729 write_vec_element_i64(tmp, v1, dst_idx, es);
730 }
731 } else {
732 /* iterate forward to avoid overwriting data we might need later */
733 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(es); dst_idx++) {
734 src_idx = (dst_idx + NUM_VEC_ELEMENTS(es)) / 2;
735 if (dst_idx % 2 == 0) {
736 read_vec_element_i64(tmp, v2, src_idx, es);
737 } else {
738 read_vec_element_i64(tmp, v3, src_idx, es);
739 }
740 write_vec_element_i64(tmp, v1, dst_idx, es);
741 }
742 }
743 tcg_temp_free_i64(tmp);
744 return DISAS_NEXT;
745 }
746
747 static DisasJumpType op_vpk(DisasContext *s, DisasOps *o)
748 {
749 const uint8_t v1 = get_field(s->fields, v1);
750 const uint8_t v2 = get_field(s->fields, v2);
751 const uint8_t v3 = get_field(s->fields, v3);
752 const uint8_t es = get_field(s->fields, m4);
753 static gen_helper_gvec_3 * const vpk[3] = {
754 gen_helper_gvec_vpk16,
755 gen_helper_gvec_vpk32,
756 gen_helper_gvec_vpk64,
757 };
758 static gen_helper_gvec_3 * const vpks[3] = {
759 gen_helper_gvec_vpks16,
760 gen_helper_gvec_vpks32,
761 gen_helper_gvec_vpks64,
762 };
763 static gen_helper_gvec_3_ptr * const vpks_cc[3] = {
764 gen_helper_gvec_vpks_cc16,
765 gen_helper_gvec_vpks_cc32,
766 gen_helper_gvec_vpks_cc64,
767 };
768 static gen_helper_gvec_3 * const vpkls[3] = {
769 gen_helper_gvec_vpkls16,
770 gen_helper_gvec_vpkls32,
771 gen_helper_gvec_vpkls64,
772 };
773 static gen_helper_gvec_3_ptr * const vpkls_cc[3] = {
774 gen_helper_gvec_vpkls_cc16,
775 gen_helper_gvec_vpkls_cc32,
776 gen_helper_gvec_vpkls_cc64,
777 };
778
779 if (es == ES_8 || es > ES_64) {
780 gen_program_exception(s, PGM_SPECIFICATION);
781 return DISAS_NORETURN;
782 }
783
784 switch (s->fields->op2) {
785 case 0x97:
786 if (get_field(s->fields, m5) & 0x1) {
787 gen_gvec_3_ptr(v1, v2, v3, cpu_env, 0, vpks_cc[es - 1]);
788 set_cc_static(s);
789 } else {
790 gen_gvec_3_ool(v1, v2, v3, 0, vpks[es - 1]);
791 }
792 break;
793 case 0x95:
794 if (get_field(s->fields, m5) & 0x1) {
795 gen_gvec_3_ptr(v1, v2, v3, cpu_env, 0, vpkls_cc[es - 1]);
796 set_cc_static(s);
797 } else {
798 gen_gvec_3_ool(v1, v2, v3, 0, vpkls[es - 1]);
799 }
800 break;
801 case 0x94:
802 /* If sources and destination dont't overlap -> fast path */
803 if (v1 != v2 && v1 != v3) {
804 const uint8_t src_es = get_field(s->fields, m4);
805 const uint8_t dst_es = src_es - 1;
806 TCGv_i64 tmp = tcg_temp_new_i64();
807 int dst_idx, src_idx;
808
809 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
810 src_idx = dst_idx;
811 if (src_idx < NUM_VEC_ELEMENTS(src_es)) {
812 read_vec_element_i64(tmp, v2, src_idx, src_es);
813 } else {
814 src_idx -= NUM_VEC_ELEMENTS(src_es);
815 read_vec_element_i64(tmp, v3, src_idx, src_es);
816 }
817 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
818 }
819 tcg_temp_free_i64(tmp);
820 } else {
821 gen_gvec_3_ool(v1, v2, v3, 0, vpk[es - 1]);
822 }
823 break;
824 default:
825 g_assert_not_reached();
826 }
827 return DISAS_NEXT;
828 }
829
830 static DisasJumpType op_vperm(DisasContext *s, DisasOps *o)
831 {
832 gen_gvec_4_ool(get_field(s->fields, v1), get_field(s->fields, v2),
833 get_field(s->fields, v3), get_field(s->fields, v4),
834 0, gen_helper_gvec_vperm);
835 return DISAS_NEXT;
836 }
837
838 static DisasJumpType op_vpdi(DisasContext *s, DisasOps *o)
839 {
840 const uint8_t i2 = extract32(get_field(s->fields, m4), 2, 1);
841 const uint8_t i3 = extract32(get_field(s->fields, m4), 0, 1);
842 TCGv_i64 t0 = tcg_temp_new_i64();
843 TCGv_i64 t1 = tcg_temp_new_i64();
844
845 read_vec_element_i64(t0, get_field(s->fields, v2), i2, ES_64);
846 read_vec_element_i64(t1, get_field(s->fields, v3), i3, ES_64);
847 write_vec_element_i64(t0, get_field(s->fields, v1), 0, ES_64);
848 write_vec_element_i64(t1, get_field(s->fields, v1), 1, ES_64);
849 tcg_temp_free_i64(t0);
850 tcg_temp_free_i64(t1);
851 return DISAS_NEXT;
852 }
853
854 static DisasJumpType op_vrep(DisasContext *s, DisasOps *o)
855 {
856 const uint8_t enr = get_field(s->fields, i2);
857 const uint8_t es = get_field(s->fields, m4);
858
859 if (es > ES_64 || !valid_vec_element(enr, es)) {
860 gen_program_exception(s, PGM_SPECIFICATION);
861 return DISAS_NORETURN;
862 }
863
864 tcg_gen_gvec_dup_mem(es, vec_full_reg_offset(get_field(s->fields, v1)),
865 vec_reg_offset(get_field(s->fields, v3), enr, es),
866 16, 16);
867 return DISAS_NEXT;
868 }
869
870 static DisasJumpType op_vrepi(DisasContext *s, DisasOps *o)
871 {
872 const int64_t data = (int16_t)get_field(s->fields, i2);
873 const uint8_t es = get_field(s->fields, m3);
874
875 if (es > ES_64) {
876 gen_program_exception(s, PGM_SPECIFICATION);
877 return DISAS_NORETURN;
878 }
879
880 gen_gvec_dupi(es, get_field(s->fields, v1), data);
881 return DISAS_NEXT;
882 }
883
884 static DisasJumpType op_vsce(DisasContext *s, DisasOps *o)
885 {
886 const uint8_t es = s->insn->data;
887 const uint8_t enr = get_field(s->fields, m3);
888 TCGv_i64 tmp;
889
890 if (!valid_vec_element(enr, es)) {
891 gen_program_exception(s, PGM_SPECIFICATION);
892 return DISAS_NORETURN;
893 }
894
895 tmp = tcg_temp_new_i64();
896 read_vec_element_i64(tmp, get_field(s->fields, v2), enr, es);
897 tcg_gen_add_i64(o->addr1, o->addr1, tmp);
898 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
899
900 read_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
901 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
902 tcg_temp_free_i64(tmp);
903 return DISAS_NEXT;
904 }
905
906 static void gen_sel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c)
907 {
908 TCGv_i64 t = tcg_temp_new_i64();
909
910 /* bit in c not set -> copy bit from b */
911 tcg_gen_andc_i64(t, b, c);
912 /* bit in c set -> copy bit from a */
913 tcg_gen_and_i64(d, a, c);
914 /* merge the results */
915 tcg_gen_or_i64(d, d, t);
916 tcg_temp_free_i64(t);
917 }
918
919 static void gen_sel_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b,
920 TCGv_vec c)
921 {
922 TCGv_vec t = tcg_temp_new_vec_matching(d);
923
924 tcg_gen_andc_vec(vece, t, b, c);
925 tcg_gen_and_vec(vece, d, a, c);
926 tcg_gen_or_vec(vece, d, d, t);
927 tcg_temp_free_vec(t);
928 }
929
930 static DisasJumpType op_vsel(DisasContext *s, DisasOps *o)
931 {
932 static const GVecGen4 gvec_op = {
933 .fni8 = gen_sel_i64,
934 .fniv = gen_sel_vec,
935 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
936 };
937
938 gen_gvec_4(get_field(s->fields, v1), get_field(s->fields, v2),
939 get_field(s->fields, v3), get_field(s->fields, v4), &gvec_op);
940 return DISAS_NEXT;
941 }
942
943 static DisasJumpType op_vseg(DisasContext *s, DisasOps *o)
944 {
945 const uint8_t es = get_field(s->fields, m3);
946 int idx1, idx2;
947 TCGv_i64 tmp;
948
949 switch (es) {
950 case ES_8:
951 idx1 = 7;
952 idx2 = 15;
953 break;
954 case ES_16:
955 idx1 = 3;
956 idx2 = 7;
957 break;
958 case ES_32:
959 idx1 = 1;
960 idx2 = 3;
961 break;
962 default:
963 gen_program_exception(s, PGM_SPECIFICATION);
964 return DISAS_NORETURN;
965 }
966
967 tmp = tcg_temp_new_i64();
968 read_vec_element_i64(tmp, get_field(s->fields, v2), idx1, es | MO_SIGN);
969 write_vec_element_i64(tmp, get_field(s->fields, v1), 0, ES_64);
970 read_vec_element_i64(tmp, get_field(s->fields, v2), idx2, es | MO_SIGN);
971 write_vec_element_i64(tmp, get_field(s->fields, v1), 1, ES_64);
972 tcg_temp_free_i64(tmp);
973 return DISAS_NEXT;
974 }
975
976 static DisasJumpType op_vst(DisasContext *s, DisasOps *o)
977 {
978 TCGv_i64 tmp = tcg_const_i64(16);
979
980 /* Probe write access before actually modifying memory */
981 gen_helper_probe_write_access(cpu_env, o->addr1, tmp);
982
983 read_vec_element_i64(tmp, get_field(s->fields, v1), 0, ES_64);
984 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
985 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
986 read_vec_element_i64(tmp, get_field(s->fields, v1), 1, ES_64);
987 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
988 tcg_temp_free_i64(tmp);
989 return DISAS_NEXT;
990 }
991
992 static DisasJumpType op_vste(DisasContext *s, DisasOps *o)
993 {
994 const uint8_t es = s->insn->data;
995 const uint8_t enr = get_field(s->fields, m3);
996 TCGv_i64 tmp;
997
998 if (!valid_vec_element(enr, es)) {
999 gen_program_exception(s, PGM_SPECIFICATION);
1000 return DISAS_NORETURN;
1001 }
1002
1003 tmp = tcg_temp_new_i64();
1004 read_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
1005 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
1006 tcg_temp_free_i64(tmp);
1007 return DISAS_NEXT;
1008 }
1009
1010 static DisasJumpType op_vstm(DisasContext *s, DisasOps *o)
1011 {
1012 const uint8_t v3 = get_field(s->fields, v3);
1013 uint8_t v1 = get_field(s->fields, v1);
1014 TCGv_i64 tmp;
1015
1016 while (v3 < v1 || (v3 - v1 + 1) > 16) {
1017 gen_program_exception(s, PGM_SPECIFICATION);
1018 return DISAS_NORETURN;
1019 }
1020
1021 /* Probe write access before actually modifying memory */
1022 tmp = tcg_const_i64((v3 - v1 + 1) * 16);
1023 gen_helper_probe_write_access(cpu_env, o->addr1, tmp);
1024
1025 for (;; v1++) {
1026 read_vec_element_i64(tmp, v1, 0, ES_64);
1027 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1028 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1029 read_vec_element_i64(tmp, v1, 1, ES_64);
1030 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1031 if (v1 == v3) {
1032 break;
1033 }
1034 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1035 }
1036 tcg_temp_free_i64(tmp);
1037 return DISAS_NEXT;
1038 }
1039
1040 static DisasJumpType op_vstl(DisasContext *s, DisasOps *o)
1041 {
1042 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
1043 TCGv_ptr a0 = tcg_temp_new_ptr();
1044
1045 /* convert highest index into an actual length */
1046 tcg_gen_addi_i64(o->in2, o->in2, 1);
1047 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
1048 gen_helper_vstl(cpu_env, a0, o->addr1, o->in2);
1049 tcg_temp_free_ptr(a0);
1050 return DISAS_NEXT;
1051 }
1052
1053 static DisasJumpType op_vup(DisasContext *s, DisasOps *o)
1054 {
1055 const bool logical = s->fields->op2 == 0xd4 || s->fields->op2 == 0xd5;
1056 const uint8_t v1 = get_field(s->fields, v1);
1057 const uint8_t v2 = get_field(s->fields, v2);
1058 const uint8_t src_es = get_field(s->fields, m3);
1059 const uint8_t dst_es = src_es + 1;
1060 int dst_idx, src_idx;
1061 TCGv_i64 tmp;
1062
1063 if (src_es > ES_32) {
1064 gen_program_exception(s, PGM_SPECIFICATION);
1065 return DISAS_NORETURN;
1066 }
1067
1068 tmp = tcg_temp_new_i64();
1069 if (s->fields->op2 == 0xd7 || s->fields->op2 == 0xd5) {
1070 /* iterate backwards to avoid overwriting data we might need later */
1071 for (dst_idx = NUM_VEC_ELEMENTS(dst_es) - 1; dst_idx >= 0; dst_idx--) {
1072 src_idx = dst_idx;
1073 read_vec_element_i64(tmp, v2, src_idx,
1074 src_es | (logical ? 0 : MO_SIGN));
1075 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1076 }
1077
1078 } else {
1079 /* iterate forward to avoid overwriting data we might need later */
1080 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
1081 src_idx = dst_idx + NUM_VEC_ELEMENTS(src_es) / 2;
1082 read_vec_element_i64(tmp, v2, src_idx,
1083 src_es | (logical ? 0 : MO_SIGN));
1084 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1085 }
1086 }
1087 tcg_temp_free_i64(tmp);
1088 return DISAS_NEXT;
1089 }
1090
1091 static DisasJumpType op_va(DisasContext *s, DisasOps *o)
1092 {
1093 const uint8_t es = get_field(s->fields, m4);
1094
1095 if (es > ES_128) {
1096 gen_program_exception(s, PGM_SPECIFICATION);
1097 return DISAS_NORETURN;
1098 } else if (es == ES_128) {
1099 gen_gvec128_3_i64(tcg_gen_add2_i64, get_field(s->fields, v1),
1100 get_field(s->fields, v2), get_field(s->fields, v3));
1101 return DISAS_NEXT;
1102 }
1103 gen_gvec_fn_3(add, es, get_field(s->fields, v1), get_field(s->fields, v2),
1104 get_field(s->fields, v3));
1105 return DISAS_NEXT;
1106 }
1107
1108 static void gen_acc(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, uint8_t es)
1109 {
1110 const uint8_t msb_bit_nr = NUM_VEC_ELEMENT_BITS(es) - 1;
1111 TCGv_i64 msb_mask = tcg_const_i64(dup_const(es, 1ull << msb_bit_nr));
1112 TCGv_i64 t1 = tcg_temp_new_i64();
1113 TCGv_i64 t2 = tcg_temp_new_i64();
1114 TCGv_i64 t3 = tcg_temp_new_i64();
1115
1116 /* Calculate the carry into the MSB, ignoring the old MSBs */
1117 tcg_gen_andc_i64(t1, a, msb_mask);
1118 tcg_gen_andc_i64(t2, b, msb_mask);
1119 tcg_gen_add_i64(t1, t1, t2);
1120 /* Calculate the MSB without any carry into it */
1121 tcg_gen_xor_i64(t3, a, b);
1122 /* Calculate the carry out of the MSB in the MSB bit position */
1123 tcg_gen_and_i64(d, a, b);
1124 tcg_gen_and_i64(t1, t1, t3);
1125 tcg_gen_or_i64(d, d, t1);
1126 /* Isolate and shift the carry into position */
1127 tcg_gen_and_i64(d, d, msb_mask);
1128 tcg_gen_shri_i64(d, d, msb_bit_nr);
1129
1130 tcg_temp_free_i64(t1);
1131 tcg_temp_free_i64(t2);
1132 tcg_temp_free_i64(t3);
1133 }
1134
1135 static void gen_acc8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1136 {
1137 gen_acc(d, a, b, ES_8);
1138 }
1139
1140 static void gen_acc16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1141 {
1142 gen_acc(d, a, b, ES_16);
1143 }
1144
1145 static void gen_acc_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1146 {
1147 TCGv_i32 t = tcg_temp_new_i32();
1148
1149 tcg_gen_add_i32(t, a, b);
1150 tcg_gen_setcond_i32(TCG_COND_LTU, d, t, b);
1151 tcg_temp_free_i32(t);
1152 }
1153
1154 static void gen_acc_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1155 {
1156 TCGv_i64 t = tcg_temp_new_i64();
1157
1158 tcg_gen_add_i64(t, a, b);
1159 tcg_gen_setcond_i64(TCG_COND_LTU, d, t, b);
1160 tcg_temp_free_i64(t);
1161 }
1162
1163 static void gen_acc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
1164 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
1165 {
1166 TCGv_i64 th = tcg_temp_new_i64();
1167 TCGv_i64 tl = tcg_temp_new_i64();
1168 TCGv_i64 zero = tcg_const_i64(0);
1169
1170 tcg_gen_add2_i64(tl, th, al, zero, bl, zero);
1171 tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1172 tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1173 tcg_gen_mov_i64(dh, zero);
1174
1175 tcg_temp_free_i64(th);
1176 tcg_temp_free_i64(tl);
1177 tcg_temp_free_i64(zero);
1178 }
1179
1180 static DisasJumpType op_vacc(DisasContext *s, DisasOps *o)
1181 {
1182 const uint8_t es = get_field(s->fields, m4);
1183 static const GVecGen3 g[4] = {
1184 { .fni8 = gen_acc8_i64, },
1185 { .fni8 = gen_acc16_i64, },
1186 { .fni4 = gen_acc_i32, },
1187 { .fni8 = gen_acc_i64, },
1188 };
1189
1190 if (es > ES_128) {
1191 gen_program_exception(s, PGM_SPECIFICATION);
1192 return DISAS_NORETURN;
1193 } else if (es == ES_128) {
1194 gen_gvec128_3_i64(gen_acc2_i64, get_field(s->fields, v1),
1195 get_field(s->fields, v2), get_field(s->fields, v3));
1196 return DISAS_NEXT;
1197 }
1198 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1199 get_field(s->fields, v3), &g[es]);
1200 return DISAS_NEXT;
1201 }
1202
1203 static void gen_ac2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1204 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1205 {
1206 TCGv_i64 tl = tcg_temp_new_i64();
1207 TCGv_i64 th = tcg_const_i64(0);
1208
1209 /* extract the carry only */
1210 tcg_gen_extract_i64(tl, cl, 0, 1);
1211 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1212 tcg_gen_add2_i64(dl, dh, dl, dh, tl, th);
1213
1214 tcg_temp_free_i64(tl);
1215 tcg_temp_free_i64(th);
1216 }
1217
1218 static DisasJumpType op_vac(DisasContext *s, DisasOps *o)
1219 {
1220 if (get_field(s->fields, m5) != ES_128) {
1221 gen_program_exception(s, PGM_SPECIFICATION);
1222 return DISAS_NORETURN;
1223 }
1224
1225 gen_gvec128_4_i64(gen_ac2_i64, get_field(s->fields, v1),
1226 get_field(s->fields, v2), get_field(s->fields, v3),
1227 get_field(s->fields, v4));
1228 return DISAS_NEXT;
1229 }
1230
1231 static void gen_accc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1232 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1233 {
1234 TCGv_i64 tl = tcg_temp_new_i64();
1235 TCGv_i64 th = tcg_temp_new_i64();
1236 TCGv_i64 zero = tcg_const_i64(0);
1237
1238 tcg_gen_andi_i64(tl, cl, 1);
1239 tcg_gen_add2_i64(tl, th, tl, zero, al, zero);
1240 tcg_gen_add2_i64(tl, th, tl, th, bl, zero);
1241 tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1242 tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1243 tcg_gen_mov_i64(dh, zero);
1244
1245 tcg_temp_free_i64(tl);
1246 tcg_temp_free_i64(th);
1247 tcg_temp_free_i64(zero);
1248 }
1249
1250 static DisasJumpType op_vaccc(DisasContext *s, DisasOps *o)
1251 {
1252 if (get_field(s->fields, m5) != ES_128) {
1253 gen_program_exception(s, PGM_SPECIFICATION);
1254 return DISAS_NORETURN;
1255 }
1256
1257 gen_gvec128_4_i64(gen_accc2_i64, get_field(s->fields, v1),
1258 get_field(s->fields, v2), get_field(s->fields, v3),
1259 get_field(s->fields, v4));
1260 return DISAS_NEXT;
1261 }
1262
1263 static DisasJumpType op_vn(DisasContext *s, DisasOps *o)
1264 {
1265 gen_gvec_fn_3(and, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1266 get_field(s->fields, v3));
1267 return DISAS_NEXT;
1268 }
1269
1270 static DisasJumpType op_vnc(DisasContext *s, DisasOps *o)
1271 {
1272 gen_gvec_fn_3(andc, ES_8, get_field(s->fields, v1),
1273 get_field(s->fields, v2), get_field(s->fields, v3));
1274 return DISAS_NEXT;
1275 }
1276
1277 static void gen_avg_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1278 {
1279 TCGv_i64 t0 = tcg_temp_new_i64();
1280 TCGv_i64 t1 = tcg_temp_new_i64();
1281
1282 tcg_gen_ext_i32_i64(t0, a);
1283 tcg_gen_ext_i32_i64(t1, b);
1284 tcg_gen_add_i64(t0, t0, t1);
1285 tcg_gen_addi_i64(t0, t0, 1);
1286 tcg_gen_shri_i64(t0, t0, 1);
1287 tcg_gen_extrl_i64_i32(d, t0);
1288
1289 tcg_temp_free(t0);
1290 tcg_temp_free(t1);
1291 }
1292
1293 static void gen_avg_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1294 {
1295 TCGv_i64 dh = tcg_temp_new_i64();
1296 TCGv_i64 ah = tcg_temp_new_i64();
1297 TCGv_i64 bh = tcg_temp_new_i64();
1298
1299 /* extending the sign by one bit is sufficient */
1300 tcg_gen_extract_i64(ah, al, 63, 1);
1301 tcg_gen_extract_i64(bh, bl, 63, 1);
1302 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1303 gen_addi2_i64(dl, dh, dl, dh, 1);
1304 tcg_gen_extract2_i64(dl, dl, dh, 1);
1305
1306 tcg_temp_free_i64(dh);
1307 tcg_temp_free_i64(ah);
1308 tcg_temp_free_i64(bh);
1309 }
1310
1311 static DisasJumpType op_vavg(DisasContext *s, DisasOps *o)
1312 {
1313 const uint8_t es = get_field(s->fields, m4);
1314 static const GVecGen3 g[4] = {
1315 { .fno = gen_helper_gvec_vavg8, },
1316 { .fno = gen_helper_gvec_vavg16, },
1317 { .fni4 = gen_avg_i32, },
1318 { .fni8 = gen_avg_i64, },
1319 };
1320
1321 if (es > ES_64) {
1322 gen_program_exception(s, PGM_SPECIFICATION);
1323 return DISAS_NORETURN;
1324 }
1325 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1326 get_field(s->fields, v3), &g[es]);
1327 return DISAS_NEXT;
1328 }
1329
1330 static void gen_avgl_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1331 {
1332 TCGv_i64 t0 = tcg_temp_new_i64();
1333 TCGv_i64 t1 = tcg_temp_new_i64();
1334
1335 tcg_gen_extu_i32_i64(t0, a);
1336 tcg_gen_extu_i32_i64(t1, b);
1337 tcg_gen_add_i64(t0, t0, t1);
1338 tcg_gen_addi_i64(t0, t0, 1);
1339 tcg_gen_shri_i64(t0, t0, 1);
1340 tcg_gen_extrl_i64_i32(d, t0);
1341
1342 tcg_temp_free(t0);
1343 tcg_temp_free(t1);
1344 }
1345
1346 static void gen_avgl_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1347 {
1348 TCGv_i64 dh = tcg_temp_new_i64();
1349 TCGv_i64 zero = tcg_const_i64(0);
1350
1351 tcg_gen_add2_i64(dl, dh, al, zero, bl, zero);
1352 gen_addi2_i64(dl, dh, dl, dh, 1);
1353 tcg_gen_extract2_i64(dl, dl, dh, 1);
1354
1355 tcg_temp_free_i64(dh);
1356 tcg_temp_free_i64(zero);
1357 }
1358
1359 static DisasJumpType op_vavgl(DisasContext *s, DisasOps *o)
1360 {
1361 const uint8_t es = get_field(s->fields, m4);
1362 static const GVecGen3 g[4] = {
1363 { .fno = gen_helper_gvec_vavgl8, },
1364 { .fno = gen_helper_gvec_vavgl16, },
1365 { .fni4 = gen_avgl_i32, },
1366 { .fni8 = gen_avgl_i64, },
1367 };
1368
1369 if (es > ES_64) {
1370 gen_program_exception(s, PGM_SPECIFICATION);
1371 return DISAS_NORETURN;
1372 }
1373 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1374 get_field(s->fields, v3), &g[es]);
1375 return DISAS_NEXT;
1376 }
1377
1378 static DisasJumpType op_vcksm(DisasContext *s, DisasOps *o)
1379 {
1380 TCGv_i32 tmp = tcg_temp_new_i32();
1381 TCGv_i32 sum = tcg_temp_new_i32();
1382 int i;
1383
1384 read_vec_element_i32(sum, get_field(s->fields, v3), 1, ES_32);
1385 for (i = 0; i < 4; i++) {
1386 read_vec_element_i32(tmp, get_field(s->fields, v2), i, ES_32);
1387 tcg_gen_add2_i32(tmp, sum, sum, sum, tmp, tmp);
1388 }
1389 zero_vec(get_field(s->fields, v1));
1390 write_vec_element_i32(sum, get_field(s->fields, v1), 1, ES_32);
1391
1392 tcg_temp_free_i32(tmp);
1393 tcg_temp_free_i32(sum);
1394 return DISAS_NEXT;
1395 }
1396
1397 static DisasJumpType op_vec(DisasContext *s, DisasOps *o)
1398 {
1399 uint8_t es = get_field(s->fields, m3);
1400 const uint8_t enr = NUM_VEC_ELEMENTS(es) / 2 - 1;
1401
1402 if (es > ES_64) {
1403 gen_program_exception(s, PGM_SPECIFICATION);
1404 return DISAS_NORETURN;
1405 }
1406 if (s->fields->op2 == 0xdb) {
1407 es |= MO_SIGN;
1408 }
1409
1410 o->in1 = tcg_temp_new_i64();
1411 o->in2 = tcg_temp_new_i64();
1412 read_vec_element_i64(o->in1, get_field(s->fields, v1), enr, es);
1413 read_vec_element_i64(o->in2, get_field(s->fields, v2), enr, es);
1414 return DISAS_NEXT;
1415 }
1416
1417 static DisasJumpType op_vc(DisasContext *s, DisasOps *o)
1418 {
1419 const uint8_t es = get_field(s->fields, m4);
1420 TCGCond cond = s->insn->data;
1421
1422 if (es > ES_64) {
1423 gen_program_exception(s, PGM_SPECIFICATION);
1424 return DISAS_NORETURN;
1425 }
1426
1427 tcg_gen_gvec_cmp(cond, es,
1428 vec_full_reg_offset(get_field(s->fields, v1)),
1429 vec_full_reg_offset(get_field(s->fields, v2)),
1430 vec_full_reg_offset(get_field(s->fields, v3)), 16, 16);
1431 if (get_field(s->fields, m5) & 0x1) {
1432 TCGv_i64 low = tcg_temp_new_i64();
1433 TCGv_i64 high = tcg_temp_new_i64();
1434
1435 read_vec_element_i64(high, get_field(s->fields, v1), 0, ES_64);
1436 read_vec_element_i64(low, get_field(s->fields, v1), 1, ES_64);
1437 gen_op_update2_cc_i64(s, CC_OP_VC, low, high);
1438
1439 tcg_temp_free_i64(low);
1440 tcg_temp_free_i64(high);
1441 }
1442 return DISAS_NEXT;
1443 }
1444
1445 static void gen_clz_i32(TCGv_i32 d, TCGv_i32 a)
1446 {
1447 tcg_gen_clzi_i32(d, a, 32);
1448 }
1449
1450 static void gen_clz_i64(TCGv_i64 d, TCGv_i64 a)
1451 {
1452 tcg_gen_clzi_i64(d, a, 64);
1453 }
1454
1455 static DisasJumpType op_vclz(DisasContext *s, DisasOps *o)
1456 {
1457 const uint8_t es = get_field(s->fields, m3);
1458 static const GVecGen2 g[4] = {
1459 { .fno = gen_helper_gvec_vclz8, },
1460 { .fno = gen_helper_gvec_vclz16, },
1461 { .fni4 = gen_clz_i32, },
1462 { .fni8 = gen_clz_i64, },
1463 };
1464
1465 if (es > ES_64) {
1466 gen_program_exception(s, PGM_SPECIFICATION);
1467 return DISAS_NORETURN;
1468 }
1469 gen_gvec_2(get_field(s->fields, v1), get_field(s->fields, v2), &g[es]);
1470 return DISAS_NEXT;
1471 }
1472
1473 static void gen_ctz_i32(TCGv_i32 d, TCGv_i32 a)
1474 {
1475 tcg_gen_ctzi_i32(d, a, 32);
1476 }
1477
1478 static void gen_ctz_i64(TCGv_i64 d, TCGv_i64 a)
1479 {
1480 tcg_gen_ctzi_i64(d, a, 64);
1481 }
1482
1483 static DisasJumpType op_vctz(DisasContext *s, DisasOps *o)
1484 {
1485 const uint8_t es = get_field(s->fields, m3);
1486 static const GVecGen2 g[4] = {
1487 { .fno = gen_helper_gvec_vctz8, },
1488 { .fno = gen_helper_gvec_vctz16, },
1489 { .fni4 = gen_ctz_i32, },
1490 { .fni8 = gen_ctz_i64, },
1491 };
1492
1493 if (es > ES_64) {
1494 gen_program_exception(s, PGM_SPECIFICATION);
1495 return DISAS_NORETURN;
1496 }
1497 gen_gvec_2(get_field(s->fields, v1), get_field(s->fields, v2), &g[es]);
1498 return DISAS_NEXT;
1499 }
1500
1501 static DisasJumpType op_vx(DisasContext *s, DisasOps *o)
1502 {
1503 gen_gvec_fn_3(xor, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1504 get_field(s->fields, v3));
1505 return DISAS_NEXT;
1506 }
1507
1508 static DisasJumpType op_vgfm(DisasContext *s, DisasOps *o)
1509 {
1510 const uint8_t es = get_field(s->fields, m4);
1511 static const GVecGen3 g[4] = {
1512 { .fno = gen_helper_gvec_vgfm8, },
1513 { .fno = gen_helper_gvec_vgfm16, },
1514 { .fno = gen_helper_gvec_vgfm32, },
1515 { .fno = gen_helper_gvec_vgfm64, },
1516 };
1517
1518 if (es > ES_64) {
1519 gen_program_exception(s, PGM_SPECIFICATION);
1520 return DISAS_NORETURN;
1521 }
1522 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1523 get_field(s->fields, v3), &g[es]);
1524 return DISAS_NEXT;
1525 }
1526
1527 static DisasJumpType op_vgfma(DisasContext *s, DisasOps *o)
1528 {
1529 const uint8_t es = get_field(s->fields, m5);
1530 static const GVecGen4 g[4] = {
1531 { .fno = gen_helper_gvec_vgfma8, },
1532 { .fno = gen_helper_gvec_vgfma16, },
1533 { .fno = gen_helper_gvec_vgfma32, },
1534 { .fno = gen_helper_gvec_vgfma64, },
1535 };
1536
1537 if (es > ES_64) {
1538 gen_program_exception(s, PGM_SPECIFICATION);
1539 return DISAS_NORETURN;
1540 }
1541 gen_gvec_4(get_field(s->fields, v1), get_field(s->fields, v2),
1542 get_field(s->fields, v3), get_field(s->fields, v4), &g[es]);
1543 return DISAS_NEXT;
1544 }
1545
1546 static DisasJumpType op_vlc(DisasContext *s, DisasOps *o)
1547 {
1548 const uint8_t es = get_field(s->fields, m3);
1549
1550 if (es > ES_64) {
1551 gen_program_exception(s, PGM_SPECIFICATION);
1552 return DISAS_NORETURN;
1553 }
1554
1555 gen_gvec_fn_2(neg, es, get_field(s->fields, v1), get_field(s->fields, v2));
1556 return DISAS_NEXT;
1557 }
1558
1559 static DisasJumpType op_vlp(DisasContext *s, DisasOps *o)
1560 {
1561 const uint8_t es = get_field(s->fields, m3);
1562
1563 if (es > ES_64) {
1564 gen_program_exception(s, PGM_SPECIFICATION);
1565 return DISAS_NORETURN;
1566 }
1567
1568 gen_gvec_fn_2(abs, es, get_field(s->fields, v1), get_field(s->fields, v2));
1569 return DISAS_NEXT;
1570 }
1571
1572 static DisasJumpType op_vmx(DisasContext *s, DisasOps *o)
1573 {
1574 const uint8_t v1 = get_field(s->fields, v1);
1575 const uint8_t v2 = get_field(s->fields, v2);
1576 const uint8_t v3 = get_field(s->fields, v3);
1577 const uint8_t es = get_field(s->fields, m4);
1578
1579 if (es > ES_64) {
1580 gen_program_exception(s, PGM_SPECIFICATION);
1581 return DISAS_NORETURN;
1582 }
1583
1584 switch (s->fields->op2) {
1585 case 0xff:
1586 gen_gvec_fn_3(smax, es, v1, v2, v3);
1587 break;
1588 case 0xfd:
1589 gen_gvec_fn_3(umax, es, v1, v2, v3);
1590 break;
1591 case 0xfe:
1592 gen_gvec_fn_3(smin, es, v1, v2, v3);
1593 break;
1594 case 0xfc:
1595 gen_gvec_fn_3(umin, es, v1, v2, v3);
1596 break;
1597 default:
1598 g_assert_not_reached();
1599 }
1600 return DISAS_NEXT;
1601 }
1602
1603 static void gen_mal_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1604 {
1605 TCGv_i32 t0 = tcg_temp_new_i32();
1606
1607 tcg_gen_mul_i32(t0, a, b);
1608 tcg_gen_add_i32(d, t0, c);
1609
1610 tcg_temp_free_i32(t0);
1611 }
1612
1613 static void gen_mah_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1614 {
1615 TCGv_i64 t0 = tcg_temp_new_i64();
1616 TCGv_i64 t1 = tcg_temp_new_i64();
1617 TCGv_i64 t2 = tcg_temp_new_i64();
1618
1619 tcg_gen_ext_i32_i64(t0, a);
1620 tcg_gen_ext_i32_i64(t1, b);
1621 tcg_gen_ext_i32_i64(t2, c);
1622 tcg_gen_mul_i64(t0, t0, t1);
1623 tcg_gen_add_i64(t0, t0, t2);
1624 tcg_gen_extrh_i64_i32(d, t0);
1625
1626 tcg_temp_free(t0);
1627 tcg_temp_free(t1);
1628 tcg_temp_free(t2);
1629 }
1630
1631 static void gen_malh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, TCGv_i32 c)
1632 {
1633 TCGv_i64 t0 = tcg_temp_new_i64();
1634 TCGv_i64 t1 = tcg_temp_new_i64();
1635 TCGv_i64 t2 = tcg_temp_new_i64();
1636
1637 tcg_gen_extu_i32_i64(t0, a);
1638 tcg_gen_extu_i32_i64(t1, b);
1639 tcg_gen_extu_i32_i64(t2, c);
1640 tcg_gen_mul_i64(t0, t0, t1);
1641 tcg_gen_add_i64(t0, t0, t2);
1642 tcg_gen_extrh_i64_i32(d, t0);
1643
1644 tcg_temp_free(t0);
1645 tcg_temp_free(t1);
1646 tcg_temp_free(t2);
1647 }
1648
1649 static DisasJumpType op_vma(DisasContext *s, DisasOps *o)
1650 {
1651 const uint8_t es = get_field(s->fields, m5);
1652 static const GVecGen4 g_vmal[3] = {
1653 { .fno = gen_helper_gvec_vmal8, },
1654 { .fno = gen_helper_gvec_vmal16, },
1655 { .fni4 = gen_mal_i32, },
1656 };
1657 static const GVecGen4 g_vmah[3] = {
1658 { .fno = gen_helper_gvec_vmah8, },
1659 { .fno = gen_helper_gvec_vmah16, },
1660 { .fni4 = gen_mah_i32, },
1661 };
1662 static const GVecGen4 g_vmalh[3] = {
1663 { .fno = gen_helper_gvec_vmalh8, },
1664 { .fno = gen_helper_gvec_vmalh16, },
1665 { .fni4 = gen_malh_i32, },
1666 };
1667 static const GVecGen4 g_vmae[3] = {
1668 { .fno = gen_helper_gvec_vmae8, },
1669 { .fno = gen_helper_gvec_vmae16, },
1670 { .fno = gen_helper_gvec_vmae32, },
1671 };
1672 static const GVecGen4 g_vmale[3] = {
1673 { .fno = gen_helper_gvec_vmale8, },
1674 { .fno = gen_helper_gvec_vmale16, },
1675 { .fno = gen_helper_gvec_vmale32, },
1676 };
1677 static const GVecGen4 g_vmao[3] = {
1678 { .fno = gen_helper_gvec_vmao8, },
1679 { .fno = gen_helper_gvec_vmao16, },
1680 { .fno = gen_helper_gvec_vmao32, },
1681 };
1682 static const GVecGen4 g_vmalo[3] = {
1683 { .fno = gen_helper_gvec_vmalo8, },
1684 { .fno = gen_helper_gvec_vmalo16, },
1685 { .fno = gen_helper_gvec_vmalo32, },
1686 };
1687 const GVecGen4 *fn;
1688
1689 if (es > ES_32) {
1690 gen_program_exception(s, PGM_SPECIFICATION);
1691 return DISAS_NORETURN;
1692 }
1693
1694 switch (s->fields->op2) {
1695 case 0xaa:
1696 fn = &g_vmal[es];
1697 break;
1698 case 0xab:
1699 fn = &g_vmah[es];
1700 break;
1701 case 0xa9:
1702 fn = &g_vmalh[es];
1703 break;
1704 case 0xae:
1705 fn = &g_vmae[es];
1706 break;
1707 case 0xac:
1708 fn = &g_vmale[es];
1709 break;
1710 case 0xaf:
1711 fn = &g_vmao[es];
1712 break;
1713 case 0xad:
1714 fn = &g_vmalo[es];
1715 break;
1716 default:
1717 g_assert_not_reached();
1718 }
1719
1720 gen_gvec_4(get_field(s->fields, v1), get_field(s->fields, v2),
1721 get_field(s->fields, v3), get_field(s->fields, v4), fn);
1722 return DISAS_NEXT;
1723 }
1724
1725 static void gen_mh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1726 {
1727 TCGv_i32 t = tcg_temp_new_i32();
1728
1729 tcg_gen_muls2_i32(t, d, a, b);
1730 tcg_temp_free_i32(t);
1731 }
1732
1733 static void gen_mlh_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1734 {
1735 TCGv_i32 t = tcg_temp_new_i32();
1736
1737 tcg_gen_mulu2_i32(t, d, a, b);
1738 tcg_temp_free_i32(t);
1739 }
1740
1741 static DisasJumpType op_vm(DisasContext *s, DisasOps *o)
1742 {
1743 const uint8_t es = get_field(s->fields, m4);
1744 static const GVecGen3 g_vmh[3] = {
1745 { .fno = gen_helper_gvec_vmh8, },
1746 { .fno = gen_helper_gvec_vmh16, },
1747 { .fni4 = gen_mh_i32, },
1748 };
1749 static const GVecGen3 g_vmlh[3] = {
1750 { .fno = gen_helper_gvec_vmlh8, },
1751 { .fno = gen_helper_gvec_vmlh16, },
1752 { .fni4 = gen_mlh_i32, },
1753 };
1754 static const GVecGen3 g_vme[3] = {
1755 { .fno = gen_helper_gvec_vme8, },
1756 { .fno = gen_helper_gvec_vme16, },
1757 { .fno = gen_helper_gvec_vme32, },
1758 };
1759 static const GVecGen3 g_vmle[3] = {
1760 { .fno = gen_helper_gvec_vmle8, },
1761 { .fno = gen_helper_gvec_vmle16, },
1762 { .fno = gen_helper_gvec_vmle32, },
1763 };
1764 static const GVecGen3 g_vmo[3] = {
1765 { .fno = gen_helper_gvec_vmo8, },
1766 { .fno = gen_helper_gvec_vmo16, },
1767 { .fno = gen_helper_gvec_vmo32, },
1768 };
1769 static const GVecGen3 g_vmlo[3] = {
1770 { .fno = gen_helper_gvec_vmlo8, },
1771 { .fno = gen_helper_gvec_vmlo16, },
1772 { .fno = gen_helper_gvec_vmlo32, },
1773 };
1774 const GVecGen3 *fn;
1775
1776 if (es > ES_32) {
1777 gen_program_exception(s, PGM_SPECIFICATION);
1778 return DISAS_NORETURN;
1779 }
1780
1781 switch (s->fields->op2) {
1782 case 0xa2:
1783 gen_gvec_fn_3(mul, es, get_field(s->fields, v1),
1784 get_field(s->fields, v2), get_field(s->fields, v3));
1785 return DISAS_NEXT;
1786 case 0xa3:
1787 fn = &g_vmh[es];
1788 break;
1789 case 0xa1:
1790 fn = &g_vmlh[es];
1791 break;
1792 case 0xa6:
1793 fn = &g_vme[es];
1794 break;
1795 case 0xa4:
1796 fn = &g_vmle[es];
1797 break;
1798 case 0xa7:
1799 fn = &g_vmo[es];
1800 break;
1801 case 0xa5:
1802 fn = &g_vmlo[es];
1803 break;
1804 default:
1805 g_assert_not_reached();
1806 }
1807
1808 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1809 get_field(s->fields, v3), fn);
1810 return DISAS_NEXT;
1811 }
1812
1813 static DisasJumpType op_vnn(DisasContext *s, DisasOps *o)
1814 {
1815 gen_gvec_fn_3(nand, ES_8, get_field(s->fields, v1),
1816 get_field(s->fields, v2), get_field(s->fields, v3));
1817 return DISAS_NEXT;
1818 }
1819
1820 static DisasJumpType op_vno(DisasContext *s, DisasOps *o)
1821 {
1822 gen_gvec_fn_3(nor, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1823 get_field(s->fields, v3));
1824 return DISAS_NEXT;
1825 }
1826
1827 static DisasJumpType op_vnx(DisasContext *s, DisasOps *o)
1828 {
1829 gen_gvec_fn_3(eqv, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1830 get_field(s->fields, v3));
1831 return DISAS_NEXT;
1832 }
1833
1834 static DisasJumpType op_vo(DisasContext *s, DisasOps *o)
1835 {
1836 gen_gvec_fn_3(or, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1837 get_field(s->fields, v3));
1838 return DISAS_NEXT;
1839 }
1840
1841 static DisasJumpType op_voc(DisasContext *s, DisasOps *o)
1842 {
1843 gen_gvec_fn_3(orc, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1844 get_field(s->fields, v3));
1845 return DISAS_NEXT;
1846 }
1847
1848 static DisasJumpType op_vpopct(DisasContext *s, DisasOps *o)
1849 {
1850 const uint8_t es = get_field(s->fields, m3);
1851 static const GVecGen2 g[4] = {
1852 { .fno = gen_helper_gvec_vpopct8, },
1853 { .fno = gen_helper_gvec_vpopct16, },
1854 { .fni4 = tcg_gen_ctpop_i32, },
1855 { .fni8 = tcg_gen_ctpop_i64, },
1856 };
1857
1858 if (es > ES_64 || (es != ES_8 && !s390_has_feat(S390_FEAT_VECTOR_ENH))) {
1859 gen_program_exception(s, PGM_SPECIFICATION);
1860 return DISAS_NORETURN;
1861 }
1862
1863 gen_gvec_2(get_field(s->fields, v1), get_field(s->fields, v2), &g[es]);
1864 return DISAS_NEXT;
1865 }
1866
1867 static void gen_rll_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1868 {
1869 TCGv_i32 t0 = tcg_temp_new_i32();
1870
1871 tcg_gen_andi_i32(t0, b, 31);
1872 tcg_gen_rotl_i32(d, a, t0);
1873 tcg_temp_free_i32(t0);
1874 }
1875
1876 static void gen_rll_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1877 {
1878 TCGv_i64 t0 = tcg_temp_new_i64();
1879
1880 tcg_gen_andi_i64(t0, b, 63);
1881 tcg_gen_rotl_i64(d, a, t0);
1882 tcg_temp_free_i64(t0);
1883 }
1884
1885 static DisasJumpType op_verllv(DisasContext *s, DisasOps *o)
1886 {
1887 const uint8_t es = get_field(s->fields, m4);
1888 static const GVecGen3 g[4] = {
1889 { .fno = gen_helper_gvec_verllv8, },
1890 { .fno = gen_helper_gvec_verllv16, },
1891 { .fni4 = gen_rll_i32, },
1892 { .fni8 = gen_rll_i64, },
1893 };
1894
1895 if (es > ES_64) {
1896 gen_program_exception(s, PGM_SPECIFICATION);
1897 return DISAS_NORETURN;
1898 }
1899
1900 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1901 get_field(s->fields, v3), &g[es]);
1902 return DISAS_NEXT;
1903 }
1904
1905 static DisasJumpType op_verll(DisasContext *s, DisasOps *o)
1906 {
1907 const uint8_t es = get_field(s->fields, m4);
1908 static const GVecGen2s g[4] = {
1909 { .fno = gen_helper_gvec_verll8, },
1910 { .fno = gen_helper_gvec_verll16, },
1911 { .fni4 = gen_rll_i32, },
1912 { .fni8 = gen_rll_i64, },
1913 };
1914
1915 if (es > ES_64) {
1916 gen_program_exception(s, PGM_SPECIFICATION);
1917 return DISAS_NORETURN;
1918 }
1919 gen_gvec_2s(get_field(s->fields, v1), get_field(s->fields, v3), o->addr1,
1920 &g[es]);
1921 return DISAS_NEXT;
1922 }
1923
1924 static void gen_rim_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b, int32_t c)
1925 {
1926 TCGv_i32 t = tcg_temp_new_i32();
1927
1928 tcg_gen_rotli_i32(t, a, c & 31);
1929 tcg_gen_and_i32(t, t, b);
1930 tcg_gen_andc_i32(d, d, b);
1931 tcg_gen_or_i32(d, d, t);
1932
1933 tcg_temp_free_i32(t);
1934 }
1935
1936 static void gen_rim_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, int64_t c)
1937 {
1938 TCGv_i64 t = tcg_temp_new_i64();
1939
1940 tcg_gen_rotli_i64(t, a, c & 63);
1941 tcg_gen_and_i64(t, t, b);
1942 tcg_gen_andc_i64(d, d, b);
1943 tcg_gen_or_i64(d, d, t);
1944
1945 tcg_temp_free_i64(t);
1946 }
1947
1948 static DisasJumpType op_verim(DisasContext *s, DisasOps *o)
1949 {
1950 const uint8_t es = get_field(s->fields, m5);
1951 const uint8_t i4 = get_field(s->fields, i4) &
1952 (NUM_VEC_ELEMENT_BITS(es) - 1);
1953 static const GVecGen3i g[4] = {
1954 { .fno = gen_helper_gvec_verim8, },
1955 { .fno = gen_helper_gvec_verim16, },
1956 { .fni4 = gen_rim_i32,
1957 .load_dest = true, },
1958 { .fni8 = gen_rim_i64,
1959 .load_dest = true, },
1960 };
1961
1962 if (es > ES_64) {
1963 gen_program_exception(s, PGM_SPECIFICATION);
1964 return DISAS_NORETURN;
1965 }
1966
1967 gen_gvec_3i(get_field(s->fields, v1), get_field(s->fields, v2),
1968 get_field(s->fields, v3), i4, &g[es]);
1969 return DISAS_NEXT;
1970 }
1971
1972 static DisasJumpType op_vesv(DisasContext *s, DisasOps *o)
1973 {
1974 const uint8_t es = get_field(s->fields, m4);
1975 const uint8_t v1 = get_field(s->fields, v1);
1976 const uint8_t v2 = get_field(s->fields, v2);
1977 const uint8_t v3 = get_field(s->fields, v3);
1978
1979 if (es > ES_64) {
1980 gen_program_exception(s, PGM_SPECIFICATION);
1981 return DISAS_NORETURN;
1982 }
1983
1984 switch (s->fields->op2) {
1985 case 0x70:
1986 gen_gvec_fn_3(shlv, es, v1, v2, v3);
1987 break;
1988 case 0x7a:
1989 gen_gvec_fn_3(sarv, es, v1, v2, v3);
1990 break;
1991 case 0x78:
1992 gen_gvec_fn_3(shrv, es, v1, v2, v3);
1993 break;
1994 default:
1995 g_assert_not_reached();
1996 }
1997 return DISAS_NEXT;
1998 }
1999
2000 static DisasJumpType op_ves(DisasContext *s, DisasOps *o)
2001 {
2002 const uint8_t es = get_field(s->fields, m4);
2003 const uint8_t d2 = get_field(s->fields, d2) &
2004 (NUM_VEC_ELEMENT_BITS(es) - 1);
2005 const uint8_t v1 = get_field(s->fields, v1);
2006 const uint8_t v3 = get_field(s->fields, v3);
2007 TCGv_i32 shift;
2008
2009 if (es > ES_64) {
2010 gen_program_exception(s, PGM_SPECIFICATION);
2011 return DISAS_NORETURN;
2012 }
2013
2014 if (likely(!get_field(s->fields, b2))) {
2015 switch (s->fields->op2) {
2016 case 0x30:
2017 gen_gvec_fn_2i(shli, es, v1, v3, d2);
2018 break;
2019 case 0x3a:
2020 gen_gvec_fn_2i(sari, es, v1, v3, d2);
2021 break;
2022 case 0x38:
2023 gen_gvec_fn_2i(shri, es, v1, v3, d2);
2024 break;
2025 default:
2026 g_assert_not_reached();
2027 }
2028 } else {
2029 shift = tcg_temp_new_i32();
2030 tcg_gen_extrl_i64_i32(shift, o->addr1);
2031 tcg_gen_andi_i32(shift, shift, NUM_VEC_ELEMENT_BITS(es) - 1);
2032 switch (s->fields->op2) {
2033 case 0x30:
2034 gen_gvec_fn_2s(shls, es, v1, v3, shift);
2035 break;
2036 case 0x3a:
2037 gen_gvec_fn_2s(sars, es, v1, v3, shift);
2038 break;
2039 case 0x38:
2040 gen_gvec_fn_2s(shrs, es, v1, v3, shift);
2041 break;
2042 default:
2043 g_assert_not_reached();
2044 }
2045 tcg_temp_free_i32(shift);
2046 }
2047 return DISAS_NEXT;
2048 }
2049
2050 static DisasJumpType op_vsl(DisasContext *s, DisasOps *o)
2051 {
2052 TCGv_i64 shift = tcg_temp_new_i64();
2053
2054 read_vec_element_i64(shift, get_field(s->fields, v3), 7, ES_8);
2055 if (s->fields->op2 == 0x74) {
2056 tcg_gen_andi_i64(shift, shift, 0x7);
2057 } else {
2058 tcg_gen_andi_i64(shift, shift, 0x78);
2059 }
2060
2061 gen_gvec_2i_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2062 shift, 0, gen_helper_gvec_vsl);
2063 tcg_temp_free_i64(shift);
2064 return DISAS_NEXT;
2065 }
2066
2067 static DisasJumpType op_vsldb(DisasContext *s, DisasOps *o)
2068 {
2069 const uint8_t i4 = get_field(s->fields, i4) & 0xf;
2070 const int left_shift = (i4 & 7) * 8;
2071 const int right_shift = 64 - left_shift;
2072 TCGv_i64 t0 = tcg_temp_new_i64();
2073 TCGv_i64 t1 = tcg_temp_new_i64();
2074 TCGv_i64 t2 = tcg_temp_new_i64();
2075
2076 if ((i4 & 8) == 0) {
2077 read_vec_element_i64(t0, get_field(s->fields, v2), 0, ES_64);
2078 read_vec_element_i64(t1, get_field(s->fields, v2), 1, ES_64);
2079 read_vec_element_i64(t2, get_field(s->fields, v3), 0, ES_64);
2080 } else {
2081 read_vec_element_i64(t0, get_field(s->fields, v2), 1, ES_64);
2082 read_vec_element_i64(t1, get_field(s->fields, v3), 0, ES_64);
2083 read_vec_element_i64(t2, get_field(s->fields, v3), 1, ES_64);
2084 }
2085 tcg_gen_extract2_i64(t0, t1, t0, right_shift);
2086 tcg_gen_extract2_i64(t1, t2, t1, right_shift);
2087 write_vec_element_i64(t0, get_field(s->fields, v1), 0, ES_64);
2088 write_vec_element_i64(t1, get_field(s->fields, v1), 1, ES_64);
2089
2090 tcg_temp_free(t0);
2091 tcg_temp_free(t1);
2092 tcg_temp_free(t2);
2093 return DISAS_NEXT;
2094 }
2095
2096 static DisasJumpType op_vsra(DisasContext *s, DisasOps *o)
2097 {
2098 TCGv_i64 shift = tcg_temp_new_i64();
2099
2100 read_vec_element_i64(shift, get_field(s->fields, v3), 7, ES_8);
2101 if (s->fields->op2 == 0x7e) {
2102 tcg_gen_andi_i64(shift, shift, 0x7);
2103 } else {
2104 tcg_gen_andi_i64(shift, shift, 0x78);
2105 }
2106
2107 gen_gvec_2i_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2108 shift, 0, gen_helper_gvec_vsra);
2109 tcg_temp_free_i64(shift);
2110 return DISAS_NEXT;
2111 }
2112
2113 static DisasJumpType op_vsrl(DisasContext *s, DisasOps *o)
2114 {
2115 TCGv_i64 shift = tcg_temp_new_i64();
2116
2117 read_vec_element_i64(shift, get_field(s->fields, v3), 7, ES_8);
2118 if (s->fields->op2 == 0x7c) {
2119 tcg_gen_andi_i64(shift, shift, 0x7);
2120 } else {
2121 tcg_gen_andi_i64(shift, shift, 0x78);
2122 }
2123
2124 gen_gvec_2i_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2125 shift, 0, gen_helper_gvec_vsrl);
2126 tcg_temp_free_i64(shift);
2127 return DISAS_NEXT;
2128 }
2129
2130 static DisasJumpType op_vs(DisasContext *s, DisasOps *o)
2131 {
2132 const uint8_t es = get_field(s->fields, m4);
2133
2134 if (es > ES_128) {
2135 gen_program_exception(s, PGM_SPECIFICATION);
2136 return DISAS_NORETURN;
2137 } else if (es == ES_128) {
2138 gen_gvec128_3_i64(tcg_gen_sub2_i64, get_field(s->fields, v1),
2139 get_field(s->fields, v2), get_field(s->fields, v3));
2140 return DISAS_NEXT;
2141 }
2142 gen_gvec_fn_3(sub, es, get_field(s->fields, v1), get_field(s->fields, v2),
2143 get_field(s->fields, v3));
2144 return DISAS_NEXT;
2145 }
2146
2147 static void gen_scbi_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2148 {
2149 tcg_gen_setcond_i32(TCG_COND_LTU, d, a, b);
2150 }
2151
2152 static void gen_scbi_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2153 {
2154 tcg_gen_setcond_i64(TCG_COND_LTU, d, a, b);
2155 }
2156
2157 static void gen_scbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
2158 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2159 {
2160 TCGv_i64 th = tcg_temp_new_i64();
2161 TCGv_i64 tl = tcg_temp_new_i64();
2162 TCGv_i64 zero = tcg_const_i64(0);
2163
2164 tcg_gen_sub2_i64(tl, th, al, zero, bl, zero);
2165 tcg_gen_andi_i64(th, th, 1);
2166 tcg_gen_sub2_i64(tl, th, ah, zero, th, zero);
2167 tcg_gen_sub2_i64(tl, th, tl, th, bh, zero);
2168 tcg_gen_andi_i64(dl, th, 1);
2169 tcg_gen_mov_i64(dh, zero);
2170
2171 tcg_temp_free_i64(th);
2172 tcg_temp_free_i64(tl);
2173 tcg_temp_free_i64(zero);
2174 }
2175
2176 static DisasJumpType op_vscbi(DisasContext *s, DisasOps *o)
2177 {
2178 const uint8_t es = get_field(s->fields, m4);
2179 static const GVecGen3 g[4] = {
2180 { .fno = gen_helper_gvec_vscbi8, },
2181 { .fno = gen_helper_gvec_vscbi16, },
2182 { .fni4 = gen_scbi_i32, },
2183 { .fni8 = gen_scbi_i64, },
2184 };
2185
2186 if (es > ES_128) {
2187 gen_program_exception(s, PGM_SPECIFICATION);
2188 return DISAS_NORETURN;
2189 } else if (es == ES_128) {
2190 gen_gvec128_3_i64(gen_scbi2_i64, get_field(s->fields, v1),
2191 get_field(s->fields, v2), get_field(s->fields, v3));
2192 return DISAS_NEXT;
2193 }
2194 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
2195 get_field(s->fields, v3), &g[es]);
2196 return DISAS_NEXT;
2197 }
2198
2199 static void gen_sbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
2200 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
2201 {
2202 TCGv_i64 tl = tcg_temp_new_i64();
2203 TCGv_i64 zero = tcg_const_i64(0);
2204
2205 tcg_gen_andi_i64(tl, cl, 1);
2206 tcg_gen_sub2_i64(dl, dh, al, ah, bl, bh);
2207 tcg_gen_sub2_i64(dl, dh, dl, dh, tl, zero);
2208 tcg_temp_free_i64(tl);
2209 tcg_temp_free_i64(zero);
2210 }
2211
2212 static DisasJumpType op_vsbi(DisasContext *s, DisasOps *o)
2213 {
2214 if (get_field(s->fields, m5) != ES_128) {
2215 gen_program_exception(s, PGM_SPECIFICATION);
2216 return DISAS_NORETURN;
2217 }
2218
2219 gen_gvec128_4_i64(gen_sbi2_i64, get_field(s->fields, v1),
2220 get_field(s->fields, v2), get_field(s->fields, v3),
2221 get_field(s->fields, v4));
2222 return DISAS_NEXT;
2223 }
2224
2225 static void gen_sbcbi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
2226 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
2227 {
2228 TCGv_i64 th = tcg_temp_new_i64();
2229 TCGv_i64 tl = tcg_temp_new_i64();
2230 TCGv_i64 zero = tcg_const_i64(0);
2231
2232 tcg_gen_andi_i64(tl, cl, 1);
2233 tcg_gen_sub2_i64(tl, th, al, zero, tl, zero);
2234 tcg_gen_sub2_i64(tl, th, tl, th, bl, zero);
2235 tcg_gen_andi_i64(th, th, 1);
2236 tcg_gen_sub2_i64(tl, th, ah, zero, th, zero);
2237 tcg_gen_sub2_i64(tl, th, tl, th, bh, zero);
2238 tcg_gen_andi_i64(dl, th, 1);
2239 tcg_gen_mov_i64(dh, zero);
2240
2241 tcg_temp_free_i64(tl);
2242 tcg_temp_free_i64(th);
2243 tcg_temp_free_i64(zero);
2244 }
2245
2246 static DisasJumpType op_vsbcbi(DisasContext *s, DisasOps *o)
2247 {
2248 if (get_field(s->fields, m5) != ES_128) {
2249 gen_program_exception(s, PGM_SPECIFICATION);
2250 return DISAS_NORETURN;
2251 }
2252
2253 gen_gvec128_4_i64(gen_sbcbi2_i64, get_field(s->fields, v1),
2254 get_field(s->fields, v2), get_field(s->fields, v3),
2255 get_field(s->fields, v4));
2256 return DISAS_NEXT;
2257 }
2258
2259 static DisasJumpType op_vsumg(DisasContext *s, DisasOps *o)
2260 {
2261 const uint8_t es = get_field(s->fields, m4);
2262 TCGv_i64 sum, tmp;
2263 uint8_t dst_idx;
2264
2265 if (es == ES_8 || es > ES_32) {
2266 gen_program_exception(s, PGM_SPECIFICATION);
2267 return DISAS_NORETURN;
2268 }
2269
2270 sum = tcg_temp_new_i64();
2271 tmp = tcg_temp_new_i64();
2272 for (dst_idx = 0; dst_idx < 2; dst_idx++) {
2273 uint8_t idx = dst_idx * NUM_VEC_ELEMENTS(es) / 2;
2274 const uint8_t max_idx = idx + NUM_VEC_ELEMENTS(es) / 2 - 1;
2275
2276 read_vec_element_i64(sum, get_field(s->fields, v3), max_idx, es);
2277 for (; idx <= max_idx; idx++) {
2278 read_vec_element_i64(tmp, get_field(s->fields, v2), idx, es);
2279 tcg_gen_add_i64(sum, sum, tmp);
2280 }
2281 write_vec_element_i64(sum, get_field(s->fields, v1), dst_idx, ES_64);
2282 }
2283 tcg_temp_free_i64(sum);
2284 tcg_temp_free_i64(tmp);
2285 return DISAS_NEXT;
2286 }
2287
2288 static DisasJumpType op_vsumq(DisasContext *s, DisasOps *o)
2289 {
2290 const uint8_t es = get_field(s->fields, m4);
2291 const uint8_t max_idx = NUM_VEC_ELEMENTS(es) - 1;
2292 TCGv_i64 sumh, suml, zero, tmpl;
2293 uint8_t idx;
2294
2295 if (es < ES_32 || es > ES_64) {
2296 gen_program_exception(s, PGM_SPECIFICATION);
2297 return DISAS_NORETURN;
2298 }
2299
2300 sumh = tcg_const_i64(0);
2301 suml = tcg_temp_new_i64();
2302 zero = tcg_const_i64(0);
2303 tmpl = tcg_temp_new_i64();
2304
2305 read_vec_element_i64(suml, get_field(s->fields, v3), max_idx, es);
2306 for (idx = 0; idx <= max_idx; idx++) {
2307 read_vec_element_i64(tmpl, get_field(s->fields, v2), idx, es);
2308 tcg_gen_add2_i64(suml, sumh, suml, sumh, tmpl, zero);
2309 }
2310 write_vec_element_i64(sumh, get_field(s->fields, v1), 0, ES_64);
2311 write_vec_element_i64(suml, get_field(s->fields, v1), 1, ES_64);
2312
2313 tcg_temp_free_i64(sumh);
2314 tcg_temp_free_i64(suml);
2315 tcg_temp_free_i64(zero);
2316 tcg_temp_free_i64(tmpl);
2317 return DISAS_NEXT;
2318 }
2319
2320 static DisasJumpType op_vsum(DisasContext *s, DisasOps *o)
2321 {
2322 const uint8_t es = get_field(s->fields, m4);
2323 TCGv_i32 sum, tmp;
2324 uint8_t dst_idx;
2325
2326 if (es > ES_16) {
2327 gen_program_exception(s, PGM_SPECIFICATION);
2328 return DISAS_NORETURN;
2329 }
2330
2331 sum = tcg_temp_new_i32();
2332 tmp = tcg_temp_new_i32();
2333 for (dst_idx = 0; dst_idx < 4; dst_idx++) {
2334 uint8_t idx = dst_idx * NUM_VEC_ELEMENTS(es) / 4;
2335 const uint8_t max_idx = idx + NUM_VEC_ELEMENTS(es) / 4 - 1;
2336
2337 read_vec_element_i32(sum, get_field(s->fields, v3), max_idx, es);
2338 for (; idx <= max_idx; idx++) {
2339 read_vec_element_i32(tmp, get_field(s->fields, v2), idx, es);
2340 tcg_gen_add_i32(sum, sum, tmp);
2341 }
2342 write_vec_element_i32(sum, get_field(s->fields, v1), dst_idx, ES_32);
2343 }
2344 tcg_temp_free_i32(sum);
2345 tcg_temp_free_i32(tmp);
2346 return DISAS_NEXT;
2347 }
2348
2349 static DisasJumpType op_vtm(DisasContext *s, DisasOps *o)
2350 {
2351 gen_gvec_2_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2352 cpu_env, 0, gen_helper_gvec_vtm);
2353 set_cc_static(s);
2354 return DISAS_NEXT;
2355 }
2356
2357 static DisasJumpType op_vfae(DisasContext *s, DisasOps *o)
2358 {
2359 const uint8_t es = get_field(s->fields, m4);
2360 const uint8_t m5 = get_field(s->fields, m5);
2361 static gen_helper_gvec_3 * const g[3] = {
2362 gen_helper_gvec_vfae8,
2363 gen_helper_gvec_vfae16,
2364 gen_helper_gvec_vfae32,
2365 };
2366 static gen_helper_gvec_3_ptr * const g_cc[3] = {
2367 gen_helper_gvec_vfae_cc8,
2368 gen_helper_gvec_vfae_cc16,
2369 gen_helper_gvec_vfae_cc32,
2370 };
2371 if (es > ES_32) {
2372 gen_program_exception(s, PGM_SPECIFICATION);
2373 return DISAS_NORETURN;
2374 }
2375
2376 if (extract32(m5, 0, 1)) {
2377 gen_gvec_3_ptr(get_field(s->fields, v1), get_field(s->fields, v2),
2378 get_field(s->fields, v3), cpu_env, m5, g_cc[es]);
2379 set_cc_static(s);
2380 } else {
2381 gen_gvec_3_ool(get_field(s->fields, v1), get_field(s->fields, v2),
2382 get_field(s->fields, v3), m5, g[es]);
2383 }
2384 return DISAS_NEXT;
2385 }
AR7 emulation for QEMU