?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
s390x/tcg: Implement VECTOR CHECKSUM
[qemu/ar7.git] / target / s390x / translate_vx.inc.c
blobb9d40ffeb7a7ad6b2426d41c649de249be1a5c35
1 /*
2 * QEMU TCG support -- s390x vector instruction translation functions
3 *
4 * Copyright (C) 2019 Red Hat Inc
5 *
6 * Authors:
7 * David Hildenbrand <david@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13 /*
14 * For most instructions that use the same element size for reads and
15 * writes, we can use real gvec vector expansion, which potantially uses
16 * real host vector instructions. As they only work up to 64 bit elements,
17 * 128 bit elements (vector is a single element) have to be handled
18 * differently. Operations that are too complicated to encode via TCG ops
19 * are handled via gvec ool (out-of-line) handlers.
20 *
21 * As soon as instructions use different element sizes for reads and writes
22 * or access elements "out of their element scope" we expand them manually
23 * in fancy loops, as gvec expansion does not deal with actual element
24 * numbers and does also not support access to other elements.
25 *
26 * 128 bit elements:
27 * As we only have i32/i64, such elements have to be loaded into two
28 * i64 values and can then be processed e.g. by tcg_gen_add2_i64.
29 *
30 * Sizes:
31 * On s390x, the operand size (oprsz) and the maximum size (maxsz) are
32 * always 16 (128 bit). What gvec code calls "vece", s390x calls "es",
33 * a.k.a. "element size". These values nicely map to MO_8 ... MO_64. Only
34 * 128 bit element size has to be treated in a special way (MO_64 + 1).
35 * We will use ES_* instead of MO_* for this reason in this file.
36 *
37 * CC handling:
38 * As gvec ool-helpers can currently not return values (besides via
39 * pointers like vectors or cpu_env), whenever we have to set the CC and
40 * can't conclude the value from the result vector, we will directly
41 * set it in "env->cc_op" and mark it as static via set_cc_static()".
42 * Whenever this is done, the helper writes globals (cc_op).
43 */
44
45 #define NUM_VEC_ELEMENT_BYTES(es) (1 << (es))
46 #define NUM_VEC_ELEMENTS(es) (16 / NUM_VEC_ELEMENT_BYTES(es))
47 #define NUM_VEC_ELEMENT_BITS(es) (NUM_VEC_ELEMENT_BYTES(es) * BITS_PER_BYTE)
48
49 #define ES_8 MO_8
50 #define ES_16 MO_16
51 #define ES_32 MO_32
52 #define ES_64 MO_64
53 #define ES_128 4
54
55 static inline bool valid_vec_element(uint8_t enr, TCGMemOp es)
56 {
57 return !(enr & ~(NUM_VEC_ELEMENTS(es) - 1));
58 }
59
60 static void read_vec_element_i64(TCGv_i64 dst, uint8_t reg, uint8_t enr,
61 TCGMemOp memop)
62 {
63 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
64
65 switch (memop) {
66 case ES_8:
67 tcg_gen_ld8u_i64(dst, cpu_env, offs);
68 break;
69 case ES_16:
70 tcg_gen_ld16u_i64(dst, cpu_env, offs);
71 break;
72 case ES_32:
73 tcg_gen_ld32u_i64(dst, cpu_env, offs);
74 break;
75 case ES_8 | MO_SIGN:
76 tcg_gen_ld8s_i64(dst, cpu_env, offs);
77 break;
78 case ES_16 | MO_SIGN:
79 tcg_gen_ld16s_i64(dst, cpu_env, offs);
80 break;
81 case ES_32 | MO_SIGN:
82 tcg_gen_ld32s_i64(dst, cpu_env, offs);
83 break;
84 case ES_64:
85 case ES_64 | MO_SIGN:
86 tcg_gen_ld_i64(dst, cpu_env, offs);
87 break;
88 default:
89 g_assert_not_reached();
90 }
91 }
92
93 static void read_vec_element_i32(TCGv_i32 dst, uint8_t reg, uint8_t enr,
94 TCGMemOp memop)
95 {
96 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
97
98 switch (memop) {
99 case ES_8:
100 tcg_gen_ld8u_i32(dst, cpu_env, offs);
101 break;
102 case ES_16:
103 tcg_gen_ld16u_i32(dst, cpu_env, offs);
104 break;
105 case ES_8 | MO_SIGN:
106 tcg_gen_ld8s_i32(dst, cpu_env, offs);
107 break;
108 case ES_16 | MO_SIGN:
109 tcg_gen_ld16s_i32(dst, cpu_env, offs);
110 break;
111 case ES_32:
112 case ES_32 | MO_SIGN:
113 tcg_gen_ld_i32(dst, cpu_env, offs);
114 break;
115 default:
116 g_assert_not_reached();
117 }
118 }
119
120 static void write_vec_element_i64(TCGv_i64 src, int reg, uint8_t enr,
121 TCGMemOp memop)
122 {
123 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
124
125 switch (memop) {
126 case ES_8:
127 tcg_gen_st8_i64(src, cpu_env, offs);
128 break;
129 case ES_16:
130 tcg_gen_st16_i64(src, cpu_env, offs);
131 break;
132 case ES_32:
133 tcg_gen_st32_i64(src, cpu_env, offs);
134 break;
135 case ES_64:
136 tcg_gen_st_i64(src, cpu_env, offs);
137 break;
138 default:
139 g_assert_not_reached();
140 }
141 }
142
143 static void write_vec_element_i32(TCGv_i32 src, int reg, uint8_t enr,
144 TCGMemOp memop)
145 {
146 const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
147
148 switch (memop) {
149 case ES_8:
150 tcg_gen_st8_i32(src, cpu_env, offs);
151 break;
152 case ES_16:
153 tcg_gen_st16_i32(src, cpu_env, offs);
154 break;
155 case ES_32:
156 tcg_gen_st_i32(src, cpu_env, offs);
157 break;
158 default:
159 g_assert_not_reached();
160 }
161 }
162
163 static void get_vec_element_ptr_i64(TCGv_ptr ptr, uint8_t reg, TCGv_i64 enr,
164 uint8_t es)
165 {
166 TCGv_i64 tmp = tcg_temp_new_i64();
167
168 /* mask off invalid parts from the element nr */
169 tcg_gen_andi_i64(tmp, enr, NUM_VEC_ELEMENTS(es) - 1);
170
171 /* convert it to an element offset relative to cpu_env (vec_reg_offset() */
172 tcg_gen_shli_i64(tmp, tmp, es);
173 #ifndef HOST_WORDS_BIGENDIAN
174 tcg_gen_xori_i64(tmp, tmp, 8 - NUM_VEC_ELEMENT_BYTES(es));
175 #endif
176 tcg_gen_addi_i64(tmp, tmp, vec_full_reg_offset(reg));
177
178 /* generate the final ptr by adding cpu_env */
179 tcg_gen_trunc_i64_ptr(ptr, tmp);
180 tcg_gen_add_ptr(ptr, ptr, cpu_env);
181
182 tcg_temp_free_i64(tmp);
183 }
184
185 #define gen_gvec_3(v1, v2, v3, gen) \
186 tcg_gen_gvec_3(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
187 vec_full_reg_offset(v3), 16, 16, gen)
188 #define gen_gvec_3_ool(v1, v2, v3, data, fn) \
189 tcg_gen_gvec_3_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
190 vec_full_reg_offset(v3), 16, 16, data, fn)
191 #define gen_gvec_3_ptr(v1, v2, v3, ptr, data, fn) \
192 tcg_gen_gvec_3_ptr(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
193 vec_full_reg_offset(v3), ptr, 16, 16, data, fn)
194 #define gen_gvec_4(v1, v2, v3, v4, gen) \
195 tcg_gen_gvec_4(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
196 vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
197 16, 16, gen)
198 #define gen_gvec_4_ool(v1, v2, v3, v4, data, fn) \
199 tcg_gen_gvec_4_ool(vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
200 vec_full_reg_offset(v3), vec_full_reg_offset(v4), \
201 16, 16, data, fn)
202 #define gen_gvec_dup_i64(es, v1, c) \
203 tcg_gen_gvec_dup_i64(es, vec_full_reg_offset(v1), 16, 16, c)
204 #define gen_gvec_mov(v1, v2) \
205 tcg_gen_gvec_mov(0, vec_full_reg_offset(v1), vec_full_reg_offset(v2), 16, \
206 16)
207 #define gen_gvec_dup64i(v1, c) \
208 tcg_gen_gvec_dup64i(vec_full_reg_offset(v1), 16, 16, c)
209 #define gen_gvec_fn_3(fn, es, v1, v2, v3) \
210 tcg_gen_gvec_##fn(es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
211 vec_full_reg_offset(v3), 16, 16)
212
213 /*
214 * Helper to carry out a 128 bit vector computation using 2 i64 values per
215 * vector.
216 */
217 typedef void (*gen_gvec128_3_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
218 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh);
219 static void gen_gvec128_3_i64(gen_gvec128_3_i64_fn fn, uint8_t d, uint8_t a,
220 uint8_t b)
221 {
222 TCGv_i64 dh = tcg_temp_new_i64();
223 TCGv_i64 dl = tcg_temp_new_i64();
224 TCGv_i64 ah = tcg_temp_new_i64();
225 TCGv_i64 al = tcg_temp_new_i64();
226 TCGv_i64 bh = tcg_temp_new_i64();
227 TCGv_i64 bl = tcg_temp_new_i64();
228
229 read_vec_element_i64(ah, a, 0, ES_64);
230 read_vec_element_i64(al, a, 1, ES_64);
231 read_vec_element_i64(bh, b, 0, ES_64);
232 read_vec_element_i64(bl, b, 1, ES_64);
233 fn(dl, dh, al, ah, bl, bh);
234 write_vec_element_i64(dh, d, 0, ES_64);
235 write_vec_element_i64(dl, d, 1, ES_64);
236
237 tcg_temp_free_i64(dh);
238 tcg_temp_free_i64(dl);
239 tcg_temp_free_i64(ah);
240 tcg_temp_free_i64(al);
241 tcg_temp_free_i64(bh);
242 tcg_temp_free_i64(bl);
243 }
244
245 typedef void (*gen_gvec128_4_i64_fn)(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
246 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh,
247 TCGv_i64 cl, TCGv_i64 ch);
248 static void gen_gvec128_4_i64(gen_gvec128_4_i64_fn fn, uint8_t d, uint8_t a,
249 uint8_t b, uint8_t c)
250 {
251 TCGv_i64 dh = tcg_temp_new_i64();
252 TCGv_i64 dl = tcg_temp_new_i64();
253 TCGv_i64 ah = tcg_temp_new_i64();
254 TCGv_i64 al = tcg_temp_new_i64();
255 TCGv_i64 bh = tcg_temp_new_i64();
256 TCGv_i64 bl = tcg_temp_new_i64();
257 TCGv_i64 ch = tcg_temp_new_i64();
258 TCGv_i64 cl = tcg_temp_new_i64();
259
260 read_vec_element_i64(ah, a, 0, ES_64);
261 read_vec_element_i64(al, a, 1, ES_64);
262 read_vec_element_i64(bh, b, 0, ES_64);
263 read_vec_element_i64(bl, b, 1, ES_64);
264 read_vec_element_i64(ch, c, 0, ES_64);
265 read_vec_element_i64(cl, c, 1, ES_64);
266 fn(dl, dh, al, ah, bl, bh, cl, ch);
267 write_vec_element_i64(dh, d, 0, ES_64);
268 write_vec_element_i64(dl, d, 1, ES_64);
269
270 tcg_temp_free_i64(dh);
271 tcg_temp_free_i64(dl);
272 tcg_temp_free_i64(ah);
273 tcg_temp_free_i64(al);
274 tcg_temp_free_i64(bh);
275 tcg_temp_free_i64(bl);
276 tcg_temp_free_i64(ch);
277 tcg_temp_free_i64(cl);
278 }
279
280 static void gen_gvec_dupi(uint8_t es, uint8_t reg, uint64_t c)
281 {
282 switch (es) {
283 case ES_8:
284 tcg_gen_gvec_dup8i(vec_full_reg_offset(reg), 16, 16, c);
285 break;
286 case ES_16:
287 tcg_gen_gvec_dup16i(vec_full_reg_offset(reg), 16, 16, c);
288 break;
289 case ES_32:
290 tcg_gen_gvec_dup32i(vec_full_reg_offset(reg), 16, 16, c);
291 break;
292 case ES_64:
293 gen_gvec_dup64i(reg, c);
294 break;
295 default:
296 g_assert_not_reached();
297 }
298 }
299
300 static void zero_vec(uint8_t reg)
301 {
302 tcg_gen_gvec_dup8i(vec_full_reg_offset(reg), 16, 16, 0);
303 }
304
305 static void gen_addi2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
306 uint64_t b)
307 {
308 TCGv_i64 bl = tcg_const_i64(b);
309 TCGv_i64 bh = tcg_const_i64(0);
310
311 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
312 tcg_temp_free_i64(bl);
313 tcg_temp_free_i64(bh);
314 }
315
316 static DisasJumpType op_vge(DisasContext *s, DisasOps *o)
317 {
318 const uint8_t es = s->insn->data;
319 const uint8_t enr = get_field(s->fields, m3);
320 TCGv_i64 tmp;
321
322 if (!valid_vec_element(enr, es)) {
323 gen_program_exception(s, PGM_SPECIFICATION);
324 return DISAS_NORETURN;
325 }
326
327 tmp = tcg_temp_new_i64();
328 read_vec_element_i64(tmp, get_field(s->fields, v2), enr, es);
329 tcg_gen_add_i64(o->addr1, o->addr1, tmp);
330 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
331
332 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
333 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
334 tcg_temp_free_i64(tmp);
335 return DISAS_NEXT;
336 }
337
338 static uint64_t generate_byte_mask(uint8_t mask)
339 {
340 uint64_t r = 0;
341 int i;
342
343 for (i = 0; i < 8; i++) {
344 if ((mask >> i) & 1) {
345 r |= 0xffull << (i * 8);
346 }
347 }
348 return r;
349 }
350
351 static DisasJumpType op_vgbm(DisasContext *s, DisasOps *o)
352 {
353 const uint16_t i2 = get_field(s->fields, i2);
354
355 if (i2 == (i2 & 0xff) * 0x0101) {
356 /*
357 * Masks for both 64 bit elements of the vector are the same.
358 * Trust tcg to produce a good constant loading.
359 */
360 gen_gvec_dup64i(get_field(s->fields, v1),
361 generate_byte_mask(i2 & 0xff));
362 } else {
363 TCGv_i64 t = tcg_temp_new_i64();
364
365 tcg_gen_movi_i64(t, generate_byte_mask(i2 >> 8));
366 write_vec_element_i64(t, get_field(s->fields, v1), 0, ES_64);
367 tcg_gen_movi_i64(t, generate_byte_mask(i2));
368 write_vec_element_i64(t, get_field(s->fields, v1), 1, ES_64);
369 tcg_temp_free_i64(t);
370 }
371 return DISAS_NEXT;
372 }
373
374 static DisasJumpType op_vgm(DisasContext *s, DisasOps *o)
375 {
376 const uint8_t es = get_field(s->fields, m4);
377 const uint8_t bits = NUM_VEC_ELEMENT_BITS(es);
378 const uint8_t i2 = get_field(s->fields, i2) & (bits - 1);
379 const uint8_t i3 = get_field(s->fields, i3) & (bits - 1);
380 uint64_t mask = 0;
381 int i;
382
383 if (es > ES_64) {
384 gen_program_exception(s, PGM_SPECIFICATION);
385 return DISAS_NORETURN;
386 }
387
388 /* generate the mask - take care of wrapping */
389 for (i = i2; ; i = (i + 1) % bits) {
390 mask |= 1ull << (bits - i - 1);
391 if (i == i3) {
392 break;
393 }
394 }
395
396 gen_gvec_dupi(es, get_field(s->fields, v1), mask);
397 return DISAS_NEXT;
398 }
399
400 static DisasJumpType op_vl(DisasContext *s, DisasOps *o)
401 {
402 TCGv_i64 t0 = tcg_temp_new_i64();
403 TCGv_i64 t1 = tcg_temp_new_i64();
404
405 tcg_gen_qemu_ld_i64(t0, o->addr1, get_mem_index(s), MO_TEQ);
406 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
407 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
408 write_vec_element_i64(t0, get_field(s->fields, v1), 0, ES_64);
409 write_vec_element_i64(t1, get_field(s->fields, v1), 1, ES_64);
410 tcg_temp_free(t0);
411 tcg_temp_free(t1);
412 return DISAS_NEXT;
413 }
414
415 static DisasJumpType op_vlr(DisasContext *s, DisasOps *o)
416 {
417 gen_gvec_mov(get_field(s->fields, v1), get_field(s->fields, v2));
418 return DISAS_NEXT;
419 }
420
421 static DisasJumpType op_vlrep(DisasContext *s, DisasOps *o)
422 {
423 const uint8_t es = get_field(s->fields, m3);
424 TCGv_i64 tmp;
425
426 if (es > ES_64) {
427 gen_program_exception(s, PGM_SPECIFICATION);
428 return DISAS_NORETURN;
429 }
430
431 tmp = tcg_temp_new_i64();
432 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
433 gen_gvec_dup_i64(es, get_field(s->fields, v1), tmp);
434 tcg_temp_free_i64(tmp);
435 return DISAS_NEXT;
436 }
437
438 static DisasJumpType op_vle(DisasContext *s, DisasOps *o)
439 {
440 const uint8_t es = s->insn->data;
441 const uint8_t enr = get_field(s->fields, m3);
442 TCGv_i64 tmp;
443
444 if (!valid_vec_element(enr, es)) {
445 gen_program_exception(s, PGM_SPECIFICATION);
446 return DISAS_NORETURN;
447 }
448
449 tmp = tcg_temp_new_i64();
450 tcg_gen_qemu_ld_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
451 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
452 tcg_temp_free_i64(tmp);
453 return DISAS_NEXT;
454 }
455
456 static DisasJumpType op_vlei(DisasContext *s, DisasOps *o)
457 {
458 const uint8_t es = s->insn->data;
459 const uint8_t enr = get_field(s->fields, m3);
460 TCGv_i64 tmp;
461
462 if (!valid_vec_element(enr, es)) {
463 gen_program_exception(s, PGM_SPECIFICATION);
464 return DISAS_NORETURN;
465 }
466
467 tmp = tcg_const_i64((int16_t)get_field(s->fields, i2));
468 write_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
469 tcg_temp_free_i64(tmp);
470 return DISAS_NEXT;
471 }
472
473 static DisasJumpType op_vlgv(DisasContext *s, DisasOps *o)
474 {
475 const uint8_t es = get_field(s->fields, m4);
476 TCGv_ptr ptr;
477
478 if (es > ES_64) {
479 gen_program_exception(s, PGM_SPECIFICATION);
480 return DISAS_NORETURN;
481 }
482
483 /* fast path if we don't need the register content */
484 if (!get_field(s->fields, b2)) {
485 uint8_t enr = get_field(s->fields, d2) & (NUM_VEC_ELEMENTS(es) - 1);
486
487 read_vec_element_i64(o->out, get_field(s->fields, v3), enr, es);
488 return DISAS_NEXT;
489 }
490
491 ptr = tcg_temp_new_ptr();
492 get_vec_element_ptr_i64(ptr, get_field(s->fields, v3), o->addr1, es);
493 switch (es) {
494 case ES_8:
495 tcg_gen_ld8u_i64(o->out, ptr, 0);
496 break;
497 case ES_16:
498 tcg_gen_ld16u_i64(o->out, ptr, 0);
499 break;
500 case ES_32:
501 tcg_gen_ld32u_i64(o->out, ptr, 0);
502 break;
503 case ES_64:
504 tcg_gen_ld_i64(o->out, ptr, 0);
505 break;
506 default:
507 g_assert_not_reached();
508 }
509 tcg_temp_free_ptr(ptr);
510
511 return DISAS_NEXT;
512 }
513
514 static DisasJumpType op_vllez(DisasContext *s, DisasOps *o)
515 {
516 uint8_t es = get_field(s->fields, m3);
517 uint8_t enr;
518 TCGv_i64 t;
519
520 switch (es) {
521 /* rightmost sub-element of leftmost doubleword */
522 case ES_8:
523 enr = 7;
524 break;
525 case ES_16:
526 enr = 3;
527 break;
528 case ES_32:
529 enr = 1;
530 break;
531 case ES_64:
532 enr = 0;
533 break;
534 /* leftmost sub-element of leftmost doubleword */
535 case 6:
536 if (s390_has_feat(S390_FEAT_VECTOR_ENH)) {
537 es = ES_32;
538 enr = 0;
539 break;
540 }
541 default:
542 /* fallthrough */
543 gen_program_exception(s, PGM_SPECIFICATION);
544 return DISAS_NORETURN;
545 }
546
547 t = tcg_temp_new_i64();
548 tcg_gen_qemu_ld_i64(t, o->addr1, get_mem_index(s), MO_TE | es);
549 zero_vec(get_field(s->fields, v1));
550 write_vec_element_i64(t, get_field(s->fields, v1), enr, es);
551 tcg_temp_free_i64(t);
552 return DISAS_NEXT;
553 }
554
555 static DisasJumpType op_vlm(DisasContext *s, DisasOps *o)
556 {
557 const uint8_t v3 = get_field(s->fields, v3);
558 uint8_t v1 = get_field(s->fields, v1);
559 TCGv_i64 t0, t1;
560
561 if (v3 < v1 || (v3 - v1 + 1) > 16) {
562 gen_program_exception(s, PGM_SPECIFICATION);
563 return DISAS_NORETURN;
564 }
565
566 /*
567 * Check for possible access exceptions by trying to load the last
568 * element. The first element will be checked first next.
569 */
570 t0 = tcg_temp_new_i64();
571 t1 = tcg_temp_new_i64();
572 gen_addi_and_wrap_i64(s, t0, o->addr1, (v3 - v1) * 16 + 8);
573 tcg_gen_qemu_ld_i64(t0, t0, get_mem_index(s), MO_TEQ);
574
575 for (;; v1++) {
576 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
577 write_vec_element_i64(t1, v1, 0, ES_64);
578 if (v1 == v3) {
579 break;
580 }
581 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
582 tcg_gen_qemu_ld_i64(t1, o->addr1, get_mem_index(s), MO_TEQ);
583 write_vec_element_i64(t1, v1, 1, ES_64);
584 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
585 }
586
587 /* Store the last element, loaded first */
588 write_vec_element_i64(t0, v1, 1, ES_64);
589
590 tcg_temp_free_i64(t0);
591 tcg_temp_free_i64(t1);
592 return DISAS_NEXT;
593 }
594
595 static DisasJumpType op_vlbb(DisasContext *s, DisasOps *o)
596 {
597 const int64_t block_size = (1ull << (get_field(s->fields, m3) + 6));
598 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
599 TCGv_ptr a0;
600 TCGv_i64 bytes;
601
602 if (get_field(s->fields, m3) > 6) {
603 gen_program_exception(s, PGM_SPECIFICATION);
604 return DISAS_NORETURN;
605 }
606
607 bytes = tcg_temp_new_i64();
608 a0 = tcg_temp_new_ptr();
609 /* calculate the number of bytes until the next block boundary */
610 tcg_gen_ori_i64(bytes, o->addr1, -block_size);
611 tcg_gen_neg_i64(bytes, bytes);
612
613 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
614 gen_helper_vll(cpu_env, a0, o->addr1, bytes);
615 tcg_temp_free_i64(bytes);
616 tcg_temp_free_ptr(a0);
617 return DISAS_NEXT;
618 }
619
620 static DisasJumpType op_vlvg(DisasContext *s, DisasOps *o)
621 {
622 const uint8_t es = get_field(s->fields, m4);
623 TCGv_ptr ptr;
624
625 if (es > ES_64) {
626 gen_program_exception(s, PGM_SPECIFICATION);
627 return DISAS_NORETURN;
628 }
629
630 /* fast path if we don't need the register content */
631 if (!get_field(s->fields, b2)) {
632 uint8_t enr = get_field(s->fields, d2) & (NUM_VEC_ELEMENTS(es) - 1);
633
634 write_vec_element_i64(o->in2, get_field(s->fields, v1), enr, es);
635 return DISAS_NEXT;
636 }
637
638 ptr = tcg_temp_new_ptr();
639 get_vec_element_ptr_i64(ptr, get_field(s->fields, v1), o->addr1, es);
640 switch (es) {
641 case ES_8:
642 tcg_gen_st8_i64(o->in2, ptr, 0);
643 break;
644 case ES_16:
645 tcg_gen_st16_i64(o->in2, ptr, 0);
646 break;
647 case ES_32:
648 tcg_gen_st32_i64(o->in2, ptr, 0);
649 break;
650 case ES_64:
651 tcg_gen_st_i64(o->in2, ptr, 0);
652 break;
653 default:
654 g_assert_not_reached();
655 }
656 tcg_temp_free_ptr(ptr);
657
658 return DISAS_NEXT;
659 }
660
661 static DisasJumpType op_vlvgp(DisasContext *s, DisasOps *o)
662 {
663 write_vec_element_i64(o->in1, get_field(s->fields, v1), 0, ES_64);
664 write_vec_element_i64(o->in2, get_field(s->fields, v1), 1, ES_64);
665 return DISAS_NEXT;
666 }
667
668 static DisasJumpType op_vll(DisasContext *s, DisasOps *o)
669 {
670 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
671 TCGv_ptr a0 = tcg_temp_new_ptr();
672
673 /* convert highest index into an actual length */
674 tcg_gen_addi_i64(o->in2, o->in2, 1);
675 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
676 gen_helper_vll(cpu_env, a0, o->addr1, o->in2);
677 tcg_temp_free_ptr(a0);
678 return DISAS_NEXT;
679 }
680
681 static DisasJumpType op_vmr(DisasContext *s, DisasOps *o)
682 {
683 const uint8_t v1 = get_field(s->fields, v1);
684 const uint8_t v2 = get_field(s->fields, v2);
685 const uint8_t v3 = get_field(s->fields, v3);
686 const uint8_t es = get_field(s->fields, m4);
687 int dst_idx, src_idx;
688 TCGv_i64 tmp;
689
690 if (es > ES_64) {
691 gen_program_exception(s, PGM_SPECIFICATION);
692 return DISAS_NORETURN;
693 }
694
695 tmp = tcg_temp_new_i64();
696 if (s->fields->op2 == 0x61) {
697 /* iterate backwards to avoid overwriting data we might need later */
698 for (dst_idx = NUM_VEC_ELEMENTS(es) - 1; dst_idx >= 0; dst_idx--) {
699 src_idx = dst_idx / 2;
700 if (dst_idx % 2 == 0) {
701 read_vec_element_i64(tmp, v2, src_idx, es);
702 } else {
703 read_vec_element_i64(tmp, v3, src_idx, es);
704 }
705 write_vec_element_i64(tmp, v1, dst_idx, es);
706 }
707 } else {
708 /* iterate forward to avoid overwriting data we might need later */
709 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(es); dst_idx++) {
710 src_idx = (dst_idx + NUM_VEC_ELEMENTS(es)) / 2;
711 if (dst_idx % 2 == 0) {
712 read_vec_element_i64(tmp, v2, src_idx, es);
713 } else {
714 read_vec_element_i64(tmp, v3, src_idx, es);
715 }
716 write_vec_element_i64(tmp, v1, dst_idx, es);
717 }
718 }
719 tcg_temp_free_i64(tmp);
720 return DISAS_NEXT;
721 }
722
723 static DisasJumpType op_vpk(DisasContext *s, DisasOps *o)
724 {
725 const uint8_t v1 = get_field(s->fields, v1);
726 const uint8_t v2 = get_field(s->fields, v2);
727 const uint8_t v3 = get_field(s->fields, v3);
728 const uint8_t es = get_field(s->fields, m4);
729 static gen_helper_gvec_3 * const vpk[3] = {
730 gen_helper_gvec_vpk16,
731 gen_helper_gvec_vpk32,
732 gen_helper_gvec_vpk64,
733 };
734 static gen_helper_gvec_3 * const vpks[3] = {
735 gen_helper_gvec_vpks16,
736 gen_helper_gvec_vpks32,
737 gen_helper_gvec_vpks64,
738 };
739 static gen_helper_gvec_3_ptr * const vpks_cc[3] = {
740 gen_helper_gvec_vpks_cc16,
741 gen_helper_gvec_vpks_cc32,
742 gen_helper_gvec_vpks_cc64,
743 };
744 static gen_helper_gvec_3 * const vpkls[3] = {
745 gen_helper_gvec_vpkls16,
746 gen_helper_gvec_vpkls32,
747 gen_helper_gvec_vpkls64,
748 };
749 static gen_helper_gvec_3_ptr * const vpkls_cc[3] = {
750 gen_helper_gvec_vpkls_cc16,
751 gen_helper_gvec_vpkls_cc32,
752 gen_helper_gvec_vpkls_cc64,
753 };
754
755 if (es == ES_8 || es > ES_64) {
756 gen_program_exception(s, PGM_SPECIFICATION);
757 return DISAS_NORETURN;
758 }
759
760 switch (s->fields->op2) {
761 case 0x97:
762 if (get_field(s->fields, m5) & 0x1) {
763 gen_gvec_3_ptr(v1, v2, v3, cpu_env, 0, vpks_cc[es - 1]);
764 set_cc_static(s);
765 } else {
766 gen_gvec_3_ool(v1, v2, v3, 0, vpks[es - 1]);
767 }
768 break;
769 case 0x95:
770 if (get_field(s->fields, m5) & 0x1) {
771 gen_gvec_3_ptr(v1, v2, v3, cpu_env, 0, vpkls_cc[es - 1]);
772 set_cc_static(s);
773 } else {
774 gen_gvec_3_ool(v1, v2, v3, 0, vpkls[es - 1]);
775 }
776 break;
777 case 0x94:
778 /* If sources and destination dont't overlap -> fast path */
779 if (v1 != v2 && v1 != v3) {
780 const uint8_t src_es = get_field(s->fields, m4);
781 const uint8_t dst_es = src_es - 1;
782 TCGv_i64 tmp = tcg_temp_new_i64();
783 int dst_idx, src_idx;
784
785 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
786 src_idx = dst_idx;
787 if (src_idx < NUM_VEC_ELEMENTS(src_es)) {
788 read_vec_element_i64(tmp, v2, src_idx, src_es);
789 } else {
790 src_idx -= NUM_VEC_ELEMENTS(src_es);
791 read_vec_element_i64(tmp, v3, src_idx, src_es);
792 }
793 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
794 }
795 tcg_temp_free_i64(tmp);
796 } else {
797 gen_gvec_3_ool(v1, v2, v3, 0, vpk[es - 1]);
798 }
799 break;
800 default:
801 g_assert_not_reached();
802 }
803 return DISAS_NEXT;
804 }
805
806 static DisasJumpType op_vperm(DisasContext *s, DisasOps *o)
807 {
808 gen_gvec_4_ool(get_field(s->fields, v1), get_field(s->fields, v2),
809 get_field(s->fields, v3), get_field(s->fields, v4),
810 0, gen_helper_gvec_vperm);
811 return DISAS_NEXT;
812 }
813
814 static DisasJumpType op_vpdi(DisasContext *s, DisasOps *o)
815 {
816 const uint8_t i2 = extract32(get_field(s->fields, m4), 2, 1);
817 const uint8_t i3 = extract32(get_field(s->fields, m4), 0, 1);
818 TCGv_i64 t0 = tcg_temp_new_i64();
819 TCGv_i64 t1 = tcg_temp_new_i64();
820
821 read_vec_element_i64(t0, get_field(s->fields, v2), i2, ES_64);
822 read_vec_element_i64(t1, get_field(s->fields, v3), i3, ES_64);
823 write_vec_element_i64(t0, get_field(s->fields, v1), 0, ES_64);
824 write_vec_element_i64(t1, get_field(s->fields, v1), 1, ES_64);
825 tcg_temp_free_i64(t0);
826 tcg_temp_free_i64(t1);
827 return DISAS_NEXT;
828 }
829
830 static DisasJumpType op_vrep(DisasContext *s, DisasOps *o)
831 {
832 const uint8_t enr = get_field(s->fields, i2);
833 const uint8_t es = get_field(s->fields, m4);
834
835 if (es > ES_64 || !valid_vec_element(enr, es)) {
836 gen_program_exception(s, PGM_SPECIFICATION);
837 return DISAS_NORETURN;
838 }
839
840 tcg_gen_gvec_dup_mem(es, vec_full_reg_offset(get_field(s->fields, v1)),
841 vec_reg_offset(get_field(s->fields, v3), enr, es),
842 16, 16);
843 return DISAS_NEXT;
844 }
845
846 static DisasJumpType op_vrepi(DisasContext *s, DisasOps *o)
847 {
848 const int64_t data = (int16_t)get_field(s->fields, i2);
849 const uint8_t es = get_field(s->fields, m3);
850
851 if (es > ES_64) {
852 gen_program_exception(s, PGM_SPECIFICATION);
853 return DISAS_NORETURN;
854 }
855
856 gen_gvec_dupi(es, get_field(s->fields, v1), data);
857 return DISAS_NEXT;
858 }
859
860 static DisasJumpType op_vsce(DisasContext *s, DisasOps *o)
861 {
862 const uint8_t es = s->insn->data;
863 const uint8_t enr = get_field(s->fields, m3);
864 TCGv_i64 tmp;
865
866 if (!valid_vec_element(enr, es)) {
867 gen_program_exception(s, PGM_SPECIFICATION);
868 return DISAS_NORETURN;
869 }
870
871 tmp = tcg_temp_new_i64();
872 read_vec_element_i64(tmp, get_field(s->fields, v2), enr, es);
873 tcg_gen_add_i64(o->addr1, o->addr1, tmp);
874 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 0);
875
876 read_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
877 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
878 tcg_temp_free_i64(tmp);
879 return DISAS_NEXT;
880 }
881
882 static void gen_sel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c)
883 {
884 TCGv_i64 t = tcg_temp_new_i64();
885
886 /* bit in c not set -> copy bit from b */
887 tcg_gen_andc_i64(t, b, c);
888 /* bit in c set -> copy bit from a */
889 tcg_gen_and_i64(d, a, c);
890 /* merge the results */
891 tcg_gen_or_i64(d, d, t);
892 tcg_temp_free_i64(t);
893 }
894
895 static void gen_sel_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b,
896 TCGv_vec c)
897 {
898 TCGv_vec t = tcg_temp_new_vec_matching(d);
899
900 tcg_gen_andc_vec(vece, t, b, c);
901 tcg_gen_and_vec(vece, d, a, c);
902 tcg_gen_or_vec(vece, d, d, t);
903 tcg_temp_free_vec(t);
904 }
905
906 static DisasJumpType op_vsel(DisasContext *s, DisasOps *o)
907 {
908 static const GVecGen4 gvec_op = {
909 .fni8 = gen_sel_i64,
910 .fniv = gen_sel_vec,
911 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
912 };
913
914 gen_gvec_4(get_field(s->fields, v1), get_field(s->fields, v2),
915 get_field(s->fields, v3), get_field(s->fields, v4), &gvec_op);
916 return DISAS_NEXT;
917 }
918
919 static DisasJumpType op_vseg(DisasContext *s, DisasOps *o)
920 {
921 const uint8_t es = get_field(s->fields, m3);
922 int idx1, idx2;
923 TCGv_i64 tmp;
924
925 switch (es) {
926 case ES_8:
927 idx1 = 7;
928 idx2 = 15;
929 break;
930 case ES_16:
931 idx1 = 3;
932 idx2 = 7;
933 break;
934 case ES_32:
935 idx1 = 1;
936 idx2 = 3;
937 break;
938 default:
939 gen_program_exception(s, PGM_SPECIFICATION);
940 return DISAS_NORETURN;
941 }
942
943 tmp = tcg_temp_new_i64();
944 read_vec_element_i64(tmp, get_field(s->fields, v2), idx1, es | MO_SIGN);
945 write_vec_element_i64(tmp, get_field(s->fields, v1), 0, ES_64);
946 read_vec_element_i64(tmp, get_field(s->fields, v2), idx2, es | MO_SIGN);
947 write_vec_element_i64(tmp, get_field(s->fields, v1), 1, ES_64);
948 tcg_temp_free_i64(tmp);
949 return DISAS_NEXT;
950 }
951
952 static DisasJumpType op_vst(DisasContext *s, DisasOps *o)
953 {
954 TCGv_i64 tmp = tcg_const_i64(16);
955
956 /* Probe write access before actually modifying memory */
957 gen_helper_probe_write_access(cpu_env, o->addr1, tmp);
958
959 read_vec_element_i64(tmp, get_field(s->fields, v1), 0, ES_64);
960 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
961 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
962 read_vec_element_i64(tmp, get_field(s->fields, v1), 1, ES_64);
963 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
964 tcg_temp_free_i64(tmp);
965 return DISAS_NEXT;
966 }
967
968 static DisasJumpType op_vste(DisasContext *s, DisasOps *o)
969 {
970 const uint8_t es = s->insn->data;
971 const uint8_t enr = get_field(s->fields, m3);
972 TCGv_i64 tmp;
973
974 if (!valid_vec_element(enr, es)) {
975 gen_program_exception(s, PGM_SPECIFICATION);
976 return DISAS_NORETURN;
977 }
978
979 tmp = tcg_temp_new_i64();
980 read_vec_element_i64(tmp, get_field(s->fields, v1), enr, es);
981 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TE | es);
982 tcg_temp_free_i64(tmp);
983 return DISAS_NEXT;
984 }
985
986 static DisasJumpType op_vstm(DisasContext *s, DisasOps *o)
987 {
988 const uint8_t v3 = get_field(s->fields, v3);
989 uint8_t v1 = get_field(s->fields, v1);
990 TCGv_i64 tmp;
991
992 while (v3 < v1 || (v3 - v1 + 1) > 16) {
993 gen_program_exception(s, PGM_SPECIFICATION);
994 return DISAS_NORETURN;
995 }
996
997 /* Probe write access before actually modifying memory */
998 tmp = tcg_const_i64((v3 - v1 + 1) * 16);
999 gen_helper_probe_write_access(cpu_env, o->addr1, tmp);
1000
1001 for (;; v1++) {
1002 read_vec_element_i64(tmp, v1, 0, ES_64);
1003 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1004 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1005 read_vec_element_i64(tmp, v1, 1, ES_64);
1006 tcg_gen_qemu_st_i64(tmp, o->addr1, get_mem_index(s), MO_TEQ);
1007 if (v1 == v3) {
1008 break;
1009 }
1010 gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
1011 }
1012 tcg_temp_free_i64(tmp);
1013 return DISAS_NEXT;
1014 }
1015
1016 static DisasJumpType op_vstl(DisasContext *s, DisasOps *o)
1017 {
1018 const int v1_offs = vec_full_reg_offset(get_field(s->fields, v1));
1019 TCGv_ptr a0 = tcg_temp_new_ptr();
1020
1021 /* convert highest index into an actual length */
1022 tcg_gen_addi_i64(o->in2, o->in2, 1);
1023 tcg_gen_addi_ptr(a0, cpu_env, v1_offs);
1024 gen_helper_vstl(cpu_env, a0, o->addr1, o->in2);
1025 tcg_temp_free_ptr(a0);
1026 return DISAS_NEXT;
1027 }
1028
1029 static DisasJumpType op_vup(DisasContext *s, DisasOps *o)
1030 {
1031 const bool logical = s->fields->op2 == 0xd4 || s->fields->op2 == 0xd5;
1032 const uint8_t v1 = get_field(s->fields, v1);
1033 const uint8_t v2 = get_field(s->fields, v2);
1034 const uint8_t src_es = get_field(s->fields, m3);
1035 const uint8_t dst_es = src_es + 1;
1036 int dst_idx, src_idx;
1037 TCGv_i64 tmp;
1038
1039 if (src_es > ES_32) {
1040 gen_program_exception(s, PGM_SPECIFICATION);
1041 return DISAS_NORETURN;
1042 }
1043
1044 tmp = tcg_temp_new_i64();
1045 if (s->fields->op2 == 0xd7 || s->fields->op2 == 0xd5) {
1046 /* iterate backwards to avoid overwriting data we might need later */
1047 for (dst_idx = NUM_VEC_ELEMENTS(dst_es) - 1; dst_idx >= 0; dst_idx--) {
1048 src_idx = dst_idx;
1049 read_vec_element_i64(tmp, v2, src_idx,
1050 src_es | (logical ? 0 : MO_SIGN));
1051 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1052 }
1053
1054 } else {
1055 /* iterate forward to avoid overwriting data we might need later */
1056 for (dst_idx = 0; dst_idx < NUM_VEC_ELEMENTS(dst_es); dst_idx++) {
1057 src_idx = dst_idx + NUM_VEC_ELEMENTS(src_es) / 2;
1058 read_vec_element_i64(tmp, v2, src_idx,
1059 src_es | (logical ? 0 : MO_SIGN));
1060 write_vec_element_i64(tmp, v1, dst_idx, dst_es);
1061 }
1062 }
1063 tcg_temp_free_i64(tmp);
1064 return DISAS_NEXT;
1065 }
1066
1067 static DisasJumpType op_va(DisasContext *s, DisasOps *o)
1068 {
1069 const uint8_t es = get_field(s->fields, m4);
1070
1071 if (es > ES_128) {
1072 gen_program_exception(s, PGM_SPECIFICATION);
1073 return DISAS_NORETURN;
1074 } else if (es == ES_128) {
1075 gen_gvec128_3_i64(tcg_gen_add2_i64, get_field(s->fields, v1),
1076 get_field(s->fields, v2), get_field(s->fields, v3));
1077 return DISAS_NEXT;
1078 }
1079 gen_gvec_fn_3(add, es, get_field(s->fields, v1), get_field(s->fields, v2),
1080 get_field(s->fields, v3));
1081 return DISAS_NEXT;
1082 }
1083
1084 static void gen_acc(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, uint8_t es)
1085 {
1086 const uint8_t msb_bit_nr = NUM_VEC_ELEMENT_BITS(es) - 1;
1087 TCGv_i64 msb_mask = tcg_const_i64(dup_const(es, 1ull << msb_bit_nr));
1088 TCGv_i64 t1 = tcg_temp_new_i64();
1089 TCGv_i64 t2 = tcg_temp_new_i64();
1090 TCGv_i64 t3 = tcg_temp_new_i64();
1091
1092 /* Calculate the carry into the MSB, ignoring the old MSBs */
1093 tcg_gen_andc_i64(t1, a, msb_mask);
1094 tcg_gen_andc_i64(t2, b, msb_mask);
1095 tcg_gen_add_i64(t1, t1, t2);
1096 /* Calculate the MSB without any carry into it */
1097 tcg_gen_xor_i64(t3, a, b);
1098 /* Calculate the carry out of the MSB in the MSB bit position */
1099 tcg_gen_and_i64(d, a, b);
1100 tcg_gen_and_i64(t1, t1, t3);
1101 tcg_gen_or_i64(d, d, t1);
1102 /* Isolate and shift the carry into position */
1103 tcg_gen_and_i64(d, d, msb_mask);
1104 tcg_gen_shri_i64(d, d, msb_bit_nr);
1105
1106 tcg_temp_free_i64(t1);
1107 tcg_temp_free_i64(t2);
1108 tcg_temp_free_i64(t3);
1109 }
1110
1111 static void gen_acc8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1112 {
1113 gen_acc(d, a, b, ES_8);
1114 }
1115
1116 static void gen_acc16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1117 {
1118 gen_acc(d, a, b, ES_16);
1119 }
1120
1121 static void gen_acc_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1122 {
1123 TCGv_i32 t = tcg_temp_new_i32();
1124
1125 tcg_gen_add_i32(t, a, b);
1126 tcg_gen_setcond_i32(TCG_COND_LTU, d, t, b);
1127 tcg_temp_free_i32(t);
1128 }
1129
1130 static void gen_acc_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1131 {
1132 TCGv_i64 t = tcg_temp_new_i64();
1133
1134 tcg_gen_add_i64(t, a, b);
1135 tcg_gen_setcond_i64(TCG_COND_LTU, d, t, b);
1136 tcg_temp_free_i64(t);
1137 }
1138
1139 static void gen_acc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al,
1140 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
1141 {
1142 TCGv_i64 th = tcg_temp_new_i64();
1143 TCGv_i64 tl = tcg_temp_new_i64();
1144 TCGv_i64 zero = tcg_const_i64(0);
1145
1146 tcg_gen_add2_i64(tl, th, al, zero, bl, zero);
1147 tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1148 tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1149 tcg_gen_mov_i64(dh, zero);
1150
1151 tcg_temp_free_i64(th);
1152 tcg_temp_free_i64(tl);
1153 tcg_temp_free_i64(zero);
1154 }
1155
1156 static DisasJumpType op_vacc(DisasContext *s, DisasOps *o)
1157 {
1158 const uint8_t es = get_field(s->fields, m4);
1159 static const GVecGen3 g[4] = {
1160 { .fni8 = gen_acc8_i64, },
1161 { .fni8 = gen_acc16_i64, },
1162 { .fni4 = gen_acc_i32, },
1163 { .fni8 = gen_acc_i64, },
1164 };
1165
1166 if (es > ES_128) {
1167 gen_program_exception(s, PGM_SPECIFICATION);
1168 return DISAS_NORETURN;
1169 } else if (es == ES_128) {
1170 gen_gvec128_3_i64(gen_acc2_i64, get_field(s->fields, v1),
1171 get_field(s->fields, v2), get_field(s->fields, v3));
1172 return DISAS_NEXT;
1173 }
1174 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1175 get_field(s->fields, v3), &g[es]);
1176 return DISAS_NEXT;
1177 }
1178
1179 static void gen_ac2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1180 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1181 {
1182 TCGv_i64 tl = tcg_temp_new_i64();
1183 TCGv_i64 th = tcg_const_i64(0);
1184
1185 /* extract the carry only */
1186 tcg_gen_extract_i64(tl, cl, 0, 1);
1187 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1188 tcg_gen_add2_i64(dl, dh, dl, dh, tl, th);
1189
1190 tcg_temp_free_i64(tl);
1191 tcg_temp_free_i64(th);
1192 }
1193
1194 static DisasJumpType op_vac(DisasContext *s, DisasOps *o)
1195 {
1196 if (get_field(s->fields, m5) != ES_128) {
1197 gen_program_exception(s, PGM_SPECIFICATION);
1198 return DISAS_NORETURN;
1199 }
1200
1201 gen_gvec128_4_i64(gen_ac2_i64, get_field(s->fields, v1),
1202 get_field(s->fields, v2), get_field(s->fields, v3),
1203 get_field(s->fields, v4));
1204 return DISAS_NEXT;
1205 }
1206
1207 static void gen_accc2_i64(TCGv_i64 dl, TCGv_i64 dh, TCGv_i64 al, TCGv_i64 ah,
1208 TCGv_i64 bl, TCGv_i64 bh, TCGv_i64 cl, TCGv_i64 ch)
1209 {
1210 TCGv_i64 tl = tcg_temp_new_i64();
1211 TCGv_i64 th = tcg_temp_new_i64();
1212 TCGv_i64 zero = tcg_const_i64(0);
1213
1214 tcg_gen_andi_i64(tl, cl, 1);
1215 tcg_gen_add2_i64(tl, th, tl, zero, al, zero);
1216 tcg_gen_add2_i64(tl, th, tl, th, bl, zero);
1217 tcg_gen_add2_i64(tl, th, th, zero, ah, zero);
1218 tcg_gen_add2_i64(tl, dl, tl, th, bh, zero);
1219 tcg_gen_mov_i64(dh, zero);
1220
1221 tcg_temp_free_i64(tl);
1222 tcg_temp_free_i64(th);
1223 tcg_temp_free_i64(zero);
1224 }
1225
1226 static DisasJumpType op_vaccc(DisasContext *s, DisasOps *o)
1227 {
1228 if (get_field(s->fields, m5) != ES_128) {
1229 gen_program_exception(s, PGM_SPECIFICATION);
1230 return DISAS_NORETURN;
1231 }
1232
1233 gen_gvec128_4_i64(gen_accc2_i64, get_field(s->fields, v1),
1234 get_field(s->fields, v2), get_field(s->fields, v3),
1235 get_field(s->fields, v4));
1236 return DISAS_NEXT;
1237 }
1238
1239 static DisasJumpType op_vn(DisasContext *s, DisasOps *o)
1240 {
1241 gen_gvec_fn_3(and, ES_8, get_field(s->fields, v1), get_field(s->fields, v2),
1242 get_field(s->fields, v3));
1243 return DISAS_NEXT;
1244 }
1245
1246 static DisasJumpType op_vnc(DisasContext *s, DisasOps *o)
1247 {
1248 gen_gvec_fn_3(andc, ES_8, get_field(s->fields, v1),
1249 get_field(s->fields, v2), get_field(s->fields, v3));
1250 return DISAS_NEXT;
1251 }
1252
1253 static void gen_avg_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1254 {
1255 TCGv_i64 t0 = tcg_temp_new_i64();
1256 TCGv_i64 t1 = tcg_temp_new_i64();
1257
1258 tcg_gen_ext_i32_i64(t0, a);
1259 tcg_gen_ext_i32_i64(t1, b);
1260 tcg_gen_add_i64(t0, t0, t1);
1261 tcg_gen_addi_i64(t0, t0, 1);
1262 tcg_gen_shri_i64(t0, t0, 1);
1263 tcg_gen_extrl_i64_i32(d, t0);
1264
1265 tcg_temp_free(t0);
1266 tcg_temp_free(t1);
1267 }
1268
1269 static void gen_avg_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1270 {
1271 TCGv_i64 dh = tcg_temp_new_i64();
1272 TCGv_i64 ah = tcg_temp_new_i64();
1273 TCGv_i64 bh = tcg_temp_new_i64();
1274
1275 /* extending the sign by one bit is sufficient */
1276 tcg_gen_extract_i64(ah, al, 63, 1);
1277 tcg_gen_extract_i64(bh, bl, 63, 1);
1278 tcg_gen_add2_i64(dl, dh, al, ah, bl, bh);
1279 gen_addi2_i64(dl, dh, dl, dh, 1);
1280 tcg_gen_extract2_i64(dl, dl, dh, 1);
1281
1282 tcg_temp_free_i64(dh);
1283 tcg_temp_free_i64(ah);
1284 tcg_temp_free_i64(bh);
1285 }
1286
1287 static DisasJumpType op_vavg(DisasContext *s, DisasOps *o)
1288 {
1289 const uint8_t es = get_field(s->fields, m4);
1290 static const GVecGen3 g[4] = {
1291 { .fno = gen_helper_gvec_vavg8, },
1292 { .fno = gen_helper_gvec_vavg16, },
1293 { .fni4 = gen_avg_i32, },
1294 { .fni8 = gen_avg_i64, },
1295 };
1296
1297 if (es > ES_64) {
1298 gen_program_exception(s, PGM_SPECIFICATION);
1299 return DISAS_NORETURN;
1300 }
1301 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1302 get_field(s->fields, v3), &g[es]);
1303 return DISAS_NEXT;
1304 }
1305
1306 static void gen_avgl_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1307 {
1308 TCGv_i64 t0 = tcg_temp_new_i64();
1309 TCGv_i64 t1 = tcg_temp_new_i64();
1310
1311 tcg_gen_extu_i32_i64(t0, a);
1312 tcg_gen_extu_i32_i64(t1, b);
1313 tcg_gen_add_i64(t0, t0, t1);
1314 tcg_gen_addi_i64(t0, t0, 1);
1315 tcg_gen_shri_i64(t0, t0, 1);
1316 tcg_gen_extrl_i64_i32(d, t0);
1317
1318 tcg_temp_free(t0);
1319 tcg_temp_free(t1);
1320 }
1321
1322 static void gen_avgl_i64(TCGv_i64 dl, TCGv_i64 al, TCGv_i64 bl)
1323 {
1324 TCGv_i64 dh = tcg_temp_new_i64();
1325 TCGv_i64 zero = tcg_const_i64(0);
1326
1327 tcg_gen_add2_i64(dl, dh, al, zero, bl, zero);
1328 gen_addi2_i64(dl, dh, dl, dh, 1);
1329 tcg_gen_extract2_i64(dl, dl, dh, 1);
1330
1331 tcg_temp_free_i64(dh);
1332 tcg_temp_free_i64(zero);
1333 }
1334
1335 static DisasJumpType op_vavgl(DisasContext *s, DisasOps *o)
1336 {
1337 const uint8_t es = get_field(s->fields, m4);
1338 static const GVecGen3 g[4] = {
1339 { .fno = gen_helper_gvec_vavgl8, },
1340 { .fno = gen_helper_gvec_vavgl16, },
1341 { .fni4 = gen_avgl_i32, },
1342 { .fni8 = gen_avgl_i64, },
1343 };
1344
1345 if (es > ES_64) {
1346 gen_program_exception(s, PGM_SPECIFICATION);
1347 return DISAS_NORETURN;
1348 }
1349 gen_gvec_3(get_field(s->fields, v1), get_field(s->fields, v2),
1350 get_field(s->fields, v3), &g[es]);
1351 return DISAS_NEXT;
1352 }
1353
1354 static DisasJumpType op_vcksm(DisasContext *s, DisasOps *o)
1355 {
1356 TCGv_i32 tmp = tcg_temp_new_i32();
1357 TCGv_i32 sum = tcg_temp_new_i32();
1358 int i;
1359
1360 read_vec_element_i32(sum, get_field(s->fields, v3), 1, ES_32);
1361 for (i = 0; i < 4; i++) {
1362 read_vec_element_i32(tmp, get_field(s->fields, v2), i, ES_32);
1363 tcg_gen_add2_i32(tmp, sum, sum, sum, tmp, tmp);
1364 }
1365 zero_vec(get_field(s->fields, v1));
1366 write_vec_element_i32(sum, get_field(s->fields, v1), 1, ES_32);
1367
1368 tcg_temp_free_i32(tmp);
1369 tcg_temp_free_i32(sum);
1370 return DISAS_NEXT;
1371 }
AR7 emulation for QEMU