| blob | 27eca591a999ee38cefb411b8f6f26ed01c74d27 |
1 /*
2 * New-style TCG opcode generator for i386 instructions
3 *
4 * Copyright (c) 2022 Red Hat, Inc.
5 *
6 * Author: Paolo Bonzini <pbonzini@redhat.com>
7 *
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 */
22 #define ZMM_OFFSET(reg) offsetof(CPUX86State, xmm_regs[reg])
24 typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg);
25 typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg);
26 typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b);
27 typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
28 TCGv_ptr reg_c);
29 typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
30 TCGv_ptr reg_c, TCGv_ptr reg_d);
31 typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
32 TCGv_i32 val);
33 typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
34 TCGv_ptr reg_c, TCGv_i32 val);
35 typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val);
36 typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c,
37 TCGv_i32 val);
38 typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
39 TCGv val);
40 typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
41 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale);
43 static inline TCGv_i32 tcg_constant8u_i32(uint8_t val)
44 {
45 return tcg_constant_i32(val);
46 }
48 static void gen_NM_exception(DisasContext *s)
49 {
50 gen_exception(s, EXCP07_PREX);
51 }
53 static void gen_illegal(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
54 {
55 gen_illegal_opcode(s);
56 }
58 static void gen_load_ea(DisasContext *s, AddressParts *mem, bool is_vsib)
59 {
60 TCGv ea = gen_lea_modrm_1(s, *mem, is_vsib);
61 gen_lea_v_seg(s, s->aflag, ea, mem->def_seg, s->override);
62 }
64 static inline int mmx_offset(MemOp ot)
65 {
66 switch (ot) {
67 case MO_8:
68 return offsetof(MMXReg, MMX_B(0));
69 case MO_16:
70 return offsetof(MMXReg, MMX_W(0));
71 case MO_32:
72 return offsetof(MMXReg, MMX_L(0));
73 case MO_64:
74 return offsetof(MMXReg, MMX_Q(0));
75 default:
76 g_assert_not_reached();
77 }
78 }
80 static inline int xmm_offset(MemOp ot)
81 {
82 switch (ot) {
83 case MO_8:
84 return offsetof(ZMMReg, ZMM_B(0));
85 case MO_16:
86 return offsetof(ZMMReg, ZMM_W(0));
87 case MO_32:
88 return offsetof(ZMMReg, ZMM_L(0));
89 case MO_64:
90 return offsetof(ZMMReg, ZMM_Q(0));
91 case MO_128:
92 return offsetof(ZMMReg, ZMM_X(0));
93 case MO_256:
94 return offsetof(ZMMReg, ZMM_Y(0));
95 default:
96 g_assert_not_reached();
97 }
98 }
100 static int vector_reg_offset(X86DecodedOp *op)
101 {
102 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE);
104 if (op->unit == X86_OP_MMX) {
105 return op->offset - mmx_offset(op->ot);
106 } else {
107 return op->offset - xmm_offset(op->ot);
108 }
109 }
111 static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n)
112 {
113 int base_ofs = vector_reg_offset(op);
114 switch(ot) {
115 case MO_8:
116 if (op->unit == X86_OP_MMX) {
117 return base_ofs + offsetof(MMXReg, MMX_B(n));
118 } else {
119 return base_ofs + offsetof(ZMMReg, ZMM_B(n));
120 }
121 case MO_16:
122 if (op->unit == X86_OP_MMX) {
123 return base_ofs + offsetof(MMXReg, MMX_W(n));
124 } else {
125 return base_ofs + offsetof(ZMMReg, ZMM_W(n));
126 }
127 case MO_32:
128 if (op->unit == X86_OP_MMX) {
129 return base_ofs + offsetof(MMXReg, MMX_L(n));
130 } else {
131 return base_ofs + offsetof(ZMMReg, ZMM_L(n));
132 }
133 case MO_64:
134 if (op->unit == X86_OP_MMX) {
135 return base_ofs;
136 } else {
137 return base_ofs + offsetof(ZMMReg, ZMM_Q(n));
138 }
139 case MO_128:
140 assert(op->unit == X86_OP_SSE);
141 return base_ofs + offsetof(ZMMReg, ZMM_X(n));
142 case MO_256:
143 assert(op->unit == X86_OP_SSE);
144 return base_ofs + offsetof(ZMMReg, ZMM_Y(n));
145 default:
146 g_assert_not_reached();
147 }
148 }
150 static void compute_mmx_offset(X86DecodedOp *op)
151 {
152 if (!op->has_ea) {
153 op->offset = offsetof(CPUX86State, fpregs[op->n].mmx) + mmx_offset(op->ot);
154 } else {
155 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot);
156 }
157 }
159 static void compute_xmm_offset(X86DecodedOp *op)
160 {
161 if (!op->has_ea) {
162 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot);
163 } else {
164 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot);
165 }
166 }
168 static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned)
169 {
170 switch(ot) {
171 case MO_8:
172 gen_op_ld_v(s, MO_8, temp, s->A0);
173 tcg_gen_st8_tl(temp, cpu_env, dest_ofs);
174 break;
175 case MO_16:
176 gen_op_ld_v(s, MO_16, temp, s->A0);
177 tcg_gen_st16_tl(temp, cpu_env, dest_ofs);
178 break;
179 case MO_32:
180 gen_op_ld_v(s, MO_32, temp, s->A0);
181 tcg_gen_st32_tl(temp, cpu_env, dest_ofs);
182 break;
183 case MO_64:
184 gen_ldq_env_A0(s, dest_ofs);
185 break;
186 case MO_128:
187 gen_ldo_env_A0(s, dest_ofs, aligned);
188 break;
189 case MO_256:
190 gen_ldy_env_A0(s, dest_ofs, aligned);
191 break;
192 default:
193 g_assert_not_reached();
194 }
195 }
197 static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot)
198 {
199 switch (decode->e.vex_class) {
200 case 2:
201 case 4:
202 if ((s->prefix & PREFIX_VEX) ||
203 decode->e.vex_special == X86_VEX_SSEUnaligned) {
204 /* MOST legacy SSE instructions require aligned memory operands, but not all. */
205 return false;
206 }
207 /* fall through */
208 case 1:
209 return ot >= MO_128;
211 default:
212 return false;
213 }
214 }
216 static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v)
217 {
218 X86DecodedOp *op = &decode->op[opn];
220 switch (op->unit) {
221 case X86_OP_SKIP:
222 return;
223 case X86_OP_SEG:
224 tcg_gen_ld32u_tl(v, cpu_env,
225 offsetof(CPUX86State,segs[op->n].selector));
226 break;
227 case X86_OP_CR:
228 tcg_gen_ld_tl(v, cpu_env, offsetof(CPUX86State, cr[op->n]));
229 break;
230 case X86_OP_DR:
231 tcg_gen_ld_tl(v, cpu_env, offsetof(CPUX86State, dr[op->n]));
232 break;
233 case X86_OP_INT:
234 if (op->has_ea) {
235 gen_op_ld_v(s, op->ot, v, s->A0);
236 } else {
237 gen_op_mov_v_reg(s, op->ot, v, op->n);
238 }
239 break;
240 case X86_OP_IMM:
241 tcg_gen_movi_tl(v, decode->immediate);
242 break;
244 case X86_OP_MMX:
245 compute_mmx_offset(op);
246 goto load_vector;
248 case X86_OP_SSE:
249 compute_xmm_offset(op);
250 load_vector:
251 if (op->has_ea) {
252 bool aligned = sse_needs_alignment(s, decode, op->ot);
253 gen_load_sse(s, v, op->ot, op->offset, aligned);
254 }
255 break;
257 default:
258 g_assert_not_reached();
259 }
260 }
262 static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn)
263 {
264 X86DecodedOp *op = &decode->op[opn];
265 if (op->v_ptr) {
266 return op->v_ptr;
267 }
268 op->v_ptr = tcg_temp_new_ptr();
270 /* The temporary points to the MMXReg or ZMMReg. */
271 tcg_gen_addi_ptr(op->v_ptr, cpu_env, vector_reg_offset(op));
272 return op->v_ptr;
273 }
275 #define OP_PTR0 op_ptr(decode, 0)
276 #define OP_PTR1 op_ptr(decode, 1)
277 #define OP_PTR2 op_ptr(decode, 2)
279 static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v)
280 {
281 X86DecodedOp *op = &decode->op[opn];
282 switch (op->unit) {
283 case X86_OP_SKIP:
284 break;
285 case X86_OP_SEG:
286 /* Note that gen_movl_seg_T0 takes care of interrupt shadow and TF. */
287 gen_movl_seg_T0(s, op->n);
288 break;
289 case X86_OP_INT:
290 if (op->has_ea) {
291 gen_op_st_v(s, op->ot, v, s->A0);
292 } else {
293 gen_op_mov_reg_v(s, op->ot, op->n, v);
294 }
295 break;
296 case X86_OP_MMX:
297 break;
298 case X86_OP_SSE:
299 if ((s->prefix & PREFIX_VEX) && op->ot == MO_128) {
300 tcg_gen_gvec_dup_imm(MO_64,
301 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)),
302 16, 16, 0);
303 }
304 break;
305 case X86_OP_CR:
306 case X86_OP_DR:
307 default:
308 g_assert_not_reached();
309 }
310 }
312 static inline int vector_len(DisasContext *s, X86DecodedInsn *decode)
313 {
314 if (decode->e.special == X86_SPECIAL_MMX &&
315 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) {
316 return 8;
317 }
318 return s->vex_l ? 32 : 16;
319 }
321 static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs)
322 {
323 MemOp ot = decode->op[0].ot;
324 int vec_len = vector_len(s, decode);
325 bool aligned = sse_needs_alignment(s, decode, ot);
327 if (!decode->op[0].has_ea) {
328 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len);
329 return;
330 }
332 switch (ot) {
333 case MO_64:
334 gen_stq_env_A0(s, src_ofs);
335 break;
336 case MO_128:
337 gen_sto_env_A0(s, src_ofs, aligned);
338 break;
339 case MO_256:
340 gen_sty_env_A0(s, src_ofs, aligned);
341 break;
342 default:
343 g_assert_not_reached();
344 }
345 }
347 static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b)
348 {
349 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b);
350 }
352 #define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1)
353 static const SSEFunc_0_epp fns_3dnow[] = {
354 [0x0c] = gen_helper_pi2fw,
355 [0x0d] = gen_helper_pi2fd,
356 [0x1c] = gen_helper_pf2iw,
357 [0x1d] = gen_helper_pf2id,
358 [0x8a] = gen_helper_pfnacc,
359 [0x8e] = gen_helper_pfpnacc,
360 [0x90] = gen_helper_pfcmpge,
361 [0x94] = gen_helper_pfmin,
362 [0x96] = gen_helper_pfrcp,
363 [0x97] = gen_helper_pfrsqrt,
364 [0x9a] = gen_helper_pfsub,
365 [0x9e] = gen_helper_pfadd,
366 [0xa0] = gen_helper_pfcmpgt,
367 [0xa4] = gen_helper_pfmax,
368 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */
369 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */
370 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */
371 [0xaa] = gen_helper_pfsubr,
372 [0xae] = gen_helper_pfacc,
373 [0xb0] = gen_helper_pfcmpeq,
374 [0xb4] = gen_helper_pfmul,
375 [0xb7] = gen_helper_pmulhrw_mmx,
376 [0xbb] = gen_helper_pswapd,
377 [0xbf] = gen_helper_pavgusb,
378 };
380 static void gen_3dnow(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
381 {
382 uint8_t b = decode->immediate;
383 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL;
385 if (!fn) {
386 gen_illegal_opcode(s);
387 return;
388 }
389 if (s->flags & HF_TS_MASK) {
390 gen_NM_exception(s);
391 return;
392 }
393 if (s->flags & HF_EM_MASK) {
394 gen_illegal_opcode(s);
395 return;
396 }
398 gen_helper_enter_mmx(cpu_env);
399 if (fn == FN_3DNOW_MOVE) {
400 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset);
401 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset);
402 } else {
403 fn(cpu_env, OP_PTR0, OP_PTR1);
404 }
405 }
407 /*
408 * 00 = v*ps Vps, Hps, Wpd
409 * 66 = v*pd Vpd, Hpd, Wps
410 * f3 = v*ss Vss, Hss, Wps
411 * f2 = v*sd Vsd, Hsd, Wps
412 */
413 static inline void gen_unary_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
414 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm,
415 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm,
416 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss)
417 {
418 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) {
419 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd;
420 if (!fn) {
421 gen_illegal_opcode(s);
422 return;
423 }
424 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
425 } else {
426 SSEFunc_0_epp ps, pd, fn;
427 ps = s->vex_l ? ps_ymm : ps_xmm;
428 pd = s->vex_l ? pd_ymm : pd_xmm;
429 fn = s->prefix & PREFIX_DATA ? pd : ps;
430 if (!fn) {
431 gen_illegal_opcode(s);
432 return;
433 }
434 fn(cpu_env, OP_PTR0, OP_PTR2);
435 }
436 }
437 #define UNARY_FP_SSE(uname, lname) \
438 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
439 { \
440 gen_unary_fp_sse(s, env, decode, \
441 gen_helper_##lname##pd_xmm, \
442 gen_helper_##lname##ps_xmm, \
443 gen_helper_##lname##pd_ymm, \
444 gen_helper_##lname##ps_ymm, \
445 gen_helper_##lname##sd, \
446 gen_helper_##lname##ss); \
447 }
448 UNARY_FP_SSE(VSQRT, sqrt)
450 /*
451 * 00 = v*ps Vps, Hps, Wpd
452 * 66 = v*pd Vpd, Hpd, Wps
453 * f3 = v*ss Vss, Hss, Wps
454 * f2 = v*sd Vsd, Hsd, Wps
455 */
456 static inline void gen_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
457 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm,
458 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm,
459 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss)
460 {
461 SSEFunc_0_eppp ps, pd, fn;
462 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) {
463 fn = s->prefix & PREFIX_REPZ ? ss : sd;
464 } else {
465 ps = s->vex_l ? ps_ymm : ps_xmm;
466 pd = s->vex_l ? pd_ymm : pd_xmm;
467 fn = s->prefix & PREFIX_DATA ? pd : ps;
468 }
469 if (fn) {
470 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
471 } else {
472 gen_illegal_opcode(s);
473 }
474 }
476 #define FP_SSE(uname, lname) \
477 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
478 { \
479 gen_fp_sse(s, env, decode, \
480 gen_helper_##lname##pd_xmm, \
481 gen_helper_##lname##ps_xmm, \
482 gen_helper_##lname##pd_ymm, \
483 gen_helper_##lname##ps_ymm, \
484 gen_helper_##lname##sd, \
485 gen_helper_##lname##ss); \
486 }
487 FP_SSE(VADD, add)
488 FP_SSE(VMUL, mul)
489 FP_SSE(VSUB, sub)
490 FP_SSE(VMIN, min)
491 FP_SSE(VDIV, div)
492 FP_SSE(VMAX, max)
494 #define FP_UNPACK_SSE(uname, lname) \
495 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
496 { \
497 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \
498 gen_fp_sse(s, env, decode, \
499 gen_helper_##lname##qdq_xmm, \
500 gen_helper_##lname##dq_xmm, \
501 gen_helper_##lname##qdq_ymm, \
502 gen_helper_##lname##dq_ymm, \
503 NULL, NULL); \
504 }
505 FP_UNPACK_SSE(VUNPCKLPx, punpckl)
506 FP_UNPACK_SSE(VUNPCKHPx, punpckh)
508 /*
509 * 00 = v*ps Vps, Wpd
510 * f3 = v*ss Vss, Wps
511 */
512 static inline void gen_unary_fp32_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
513 SSEFunc_0_epp ps_xmm,
514 SSEFunc_0_epp ps_ymm,
515 SSEFunc_0_eppp ss)
516 {
517 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) {
518 goto illegal_op;
519 } else if (s->prefix & PREFIX_REPZ) {
520 if (!ss) {
521 goto illegal_op;
522 }
523 ss(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
524 } else {
525 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm;
526 if (!fn) {
527 goto illegal_op;
528 }
529 fn(cpu_env, OP_PTR0, OP_PTR2);
530 }
531 return;
533 illegal_op:
534 gen_illegal_opcode(s);
535 }
536 #define UNARY_FP32_SSE(uname, lname) \
537 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
538 { \
539 gen_unary_fp32_sse(s, env, decode, \
540 gen_helper_##lname##ps_xmm, \
541 gen_helper_##lname##ps_ymm, \
542 gen_helper_##lname##ss); \
543 }
544 UNARY_FP32_SSE(VRSQRT, rsqrt)
545 UNARY_FP32_SSE(VRCP, rcp)
547 /*
548 * 66 = v*pd Vpd, Hpd, Wpd
549 * f2 = v*ps Vps, Hps, Wps
550 */
551 static inline void gen_horizontal_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
552 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm,
553 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm)
554 {
555 SSEFunc_0_eppp ps, pd, fn;
556 ps = s->vex_l ? ps_ymm : ps_xmm;
557 pd = s->vex_l ? pd_ymm : pd_xmm;
558 fn = s->prefix & PREFIX_DATA ? pd : ps;
559 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
560 }
561 #define HORIZONTAL_FP_SSE(uname, lname) \
562 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
563 { \
564 gen_horizontal_fp_sse(s, env, decode, \
565 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \
566 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \
567 }
568 HORIZONTAL_FP_SSE(VHADD, hadd)
569 HORIZONTAL_FP_SSE(VHSUB, hsub)
570 HORIZONTAL_FP_SSE(VADDSUB, addsub)
572 static inline void gen_ternary_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
573 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm)
574 {
575 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm;
576 TCGv_ptr ptr3 = tcg_temp_new_ptr();
578 /* The format of the fourth input is Lx */
579 tcg_gen_addi_ptr(ptr3, cpu_env, ZMM_OFFSET(op3));
580 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3);
581 tcg_temp_free_ptr(ptr3);
582 }
583 #define TERNARY_SSE(uname, uvname, lname) \
584 static void gen_##uvname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
585 { \
586 gen_ternary_sse(s, env, decode, (uint8_t)decode->immediate >> 4, \
587 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \
588 } \
589 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
590 { \
591 gen_ternary_sse(s, env, decode, 0, \
592 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \
593 }
594 TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps)
595 TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd)
596 TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb)
598 static inline void gen_binary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
599 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm)
600 {
601 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
602 if (!s->vex_l) {
603 xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
604 } else {
605 ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
606 }
607 }
609 #define BINARY_IMM_SSE(uname, lname) \
610 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
611 { \
612 gen_binary_imm_sse(s, env, decode, \
613 gen_helper_##lname##_xmm, \
614 gen_helper_##lname##_ymm); \
615 }
617 BINARY_IMM_SSE(VBLENDPD, blendpd)
618 BINARY_IMM_SSE(VBLENDPS, blendps)
619 BINARY_IMM_SSE(VPBLENDW, pblendw)
620 BINARY_IMM_SSE(VDDPS, dpps)
621 #define gen_helper_dppd_ymm NULL
622 BINARY_IMM_SSE(VDDPD, dppd)
623 BINARY_IMM_SSE(VMPSADBW, mpsadbw)
624 BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq)
627 #define UNARY_INT_GVEC(uname, func, ...) \
628 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
629 { \
630 int vec_len = vector_len(s, decode); \
631 \
632 func(__VA_ARGS__, decode->op[0].offset, \
633 decode->op[2].offset, vec_len, vec_len); \
634 }
635 UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8)
636 UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16)
637 UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32)
638 UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128)
639 UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8)
640 UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16)
641 UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32)
642 UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64)
645 #define BINARY_INT_GVEC(uname, func, ...) \
646 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
647 { \
648 int vec_len = vector_len(s, decode); \
649 \
650 func(__VA_ARGS__, \
651 decode->op[0].offset, decode->op[1].offset, \
652 decode->op[2].offset, vec_len, vec_len); \
653 }
655 BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8)
656 BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16)
657 BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32)
658 BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64)
659 BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8)
660 BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16)
661 BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8)
662 BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16)
663 BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64)
664 BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8)
665 BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32)
666 BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16)
667 BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64)
668 BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8)
669 BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16)
670 BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32)
671 BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64)
672 BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8)
673 BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16)
674 BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32)
675 BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8)
676 BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16)
677 BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32)
678 BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8)
679 BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16)
680 BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32)
681 BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8)
682 BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16)
683 BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32)
684 BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16)
685 BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32)
686 BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64)
687 BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8)
688 BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16)
689 BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32)
690 BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64)
691 BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8)
692 BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16)
693 BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8)
694 BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16)
695 BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64)
698 /*
699 * 00 = p* Pq, Qq (if mmx not NULL; no VEX)
700 * 66 = vp* Vx, Hx, Wx
701 *
702 * These are really the same encoding, because 1) V is the same as P when VEX.V
703 * is not present 2) P and Q are the same as H and W apart from MM/XMM
704 */
705 static inline void gen_binary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
706 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm)
707 {
708 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX));
710 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) {
711 /* VEX encoding is not applicable to MMX instructions. */
712 gen_illegal_opcode(s);
713 return;
714 }
715 if (!(s->prefix & PREFIX_DATA)) {
716 mmx(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
717 } else if (!s->vex_l) {
718 xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
719 } else {
720 ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
721 }
722 }
725 #define BINARY_INT_MMX(uname, lname) \
726 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
727 { \
728 gen_binary_int_sse(s, env, decode, \
729 gen_helper_##lname##_mmx, \
730 gen_helper_##lname##_xmm, \
731 gen_helper_##lname##_ymm); \
732 }
733 BINARY_INT_MMX(PUNPCKLBW, punpcklbw)
734 BINARY_INT_MMX(PUNPCKLWD, punpcklwd)
735 BINARY_INT_MMX(PUNPCKLDQ, punpckldq)
736 BINARY_INT_MMX(PACKSSWB, packsswb)
737 BINARY_INT_MMX(PACKUSWB, packuswb)
738 BINARY_INT_MMX(PUNPCKHBW, punpckhbw)
739 BINARY_INT_MMX(PUNPCKHWD, punpckhwd)
740 BINARY_INT_MMX(PUNPCKHDQ, punpckhdq)
741 BINARY_INT_MMX(PACKSSDW, packssdw)
743 BINARY_INT_MMX(PAVGB, pavgb)
744 BINARY_INT_MMX(PAVGW, pavgw)
745 BINARY_INT_MMX(PMADDWD, pmaddwd)
746 BINARY_INT_MMX(PMULHUW, pmulhuw)
747 BINARY_INT_MMX(PMULHW, pmulhw)
748 BINARY_INT_MMX(PMULUDQ, pmuludq)
749 BINARY_INT_MMX(PSADBW, psadbw)
751 BINARY_INT_MMX(PSLLW_r, psllw)
752 BINARY_INT_MMX(PSLLD_r, pslld)
753 BINARY_INT_MMX(PSLLQ_r, psllq)
754 BINARY_INT_MMX(PSRLW_r, psrlw)
755 BINARY_INT_MMX(PSRLD_r, psrld)
756 BINARY_INT_MMX(PSRLQ_r, psrlq)
757 BINARY_INT_MMX(PSRAW_r, psraw)
758 BINARY_INT_MMX(PSRAD_r, psrad)
760 BINARY_INT_MMX(PHADDW, phaddw)
761 BINARY_INT_MMX(PHADDSW, phaddsw)
762 BINARY_INT_MMX(PHADDD, phaddd)
763 BINARY_INT_MMX(PHSUBW, phsubw)
764 BINARY_INT_MMX(PHSUBSW, phsubsw)
765 BINARY_INT_MMX(PHSUBD, phsubd)
766 BINARY_INT_MMX(PMADDUBSW, pmaddubsw)
767 BINARY_INT_MMX(PSHUFB, pshufb)
768 BINARY_INT_MMX(PSIGNB, psignb)
769 BINARY_INT_MMX(PSIGNW, psignw)
770 BINARY_INT_MMX(PSIGND, psignd)
771 BINARY_INT_MMX(PMULHRSW, pmulhrsw)
773 /* Instructions with no MMX equivalent. */
774 #define BINARY_INT_SSE(uname, lname) \
775 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
776 { \
777 gen_binary_int_sse(s, env, decode, \
778 NULL, \
779 gen_helper_##lname##_xmm, \
780 gen_helper_##lname##_ymm); \
781 }
783 /* Instructions with no MMX equivalent. */
784 BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq)
785 BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq)
786 BINARY_INT_SSE(VPACKUSDW, packusdw)
787 BINARY_INT_SSE(VPERMILPS, vpermilps)
788 BINARY_INT_SSE(VPERMILPD, vpermilpd)
789 BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd)
790 BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq)
792 BINARY_INT_SSE(PMULDQ, pmuldq)
794 BINARY_INT_SSE(VAESDEC, aesdec)
795 BINARY_INT_SSE(VAESDECLAST, aesdeclast)
796 BINARY_INT_SSE(VAESENC, aesenc)
797 BINARY_INT_SSE(VAESENCLAST, aesenclast)
799 #define UNARY_CMP_SSE(uname, lname) \
800 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
801 { \
802 if (!s->vex_l) { \
803 gen_helper_##lname##_xmm(cpu_env, OP_PTR1, OP_PTR2); \
804 } else { \
805 gen_helper_##lname##_ymm(cpu_env, OP_PTR1, OP_PTR2); \
806 } \
807 set_cc_op(s, CC_OP_EFLAGS); \
808 }
809 UNARY_CMP_SSE(VPTEST, ptest)
810 UNARY_CMP_SSE(VTESTPS, vtestps)
811 UNARY_CMP_SSE(VTESTPD, vtestpd)
813 static inline void gen_unary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
814 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm)
815 {
816 if (!s->vex_l) {
817 xmm(cpu_env, OP_PTR0, OP_PTR2);
818 } else {
819 ymm(cpu_env, OP_PTR0, OP_PTR2);
820 }
821 }
823 #define UNARY_INT_SSE(uname, lname) \
824 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
825 { \
826 gen_unary_int_sse(s, env, decode, \
827 gen_helper_##lname##_xmm, \
828 gen_helper_##lname##_ymm); \
829 }
831 UNARY_INT_SSE(VPMOVSXBW, pmovsxbw)
832 UNARY_INT_SSE(VPMOVSXBD, pmovsxbd)
833 UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq)
834 UNARY_INT_SSE(VPMOVSXWD, pmovsxwd)
835 UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq)
836 UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq)
838 UNARY_INT_SSE(VPMOVZXBW, pmovzxbw)
839 UNARY_INT_SSE(VPMOVZXBD, pmovzxbd)
840 UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq)
841 UNARY_INT_SSE(VPMOVZXWD, pmovzxwd)
842 UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq)
843 UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq)
845 UNARY_INT_SSE(VMOVSLDUP, pmovsldup)
846 UNARY_INT_SSE(VMOVSHDUP, pmovshdup)
847 UNARY_INT_SSE(VMOVDDUP, pmovdldup)
849 UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd)
850 UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq)
851 UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq)
852 UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps)
853 UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq)
854 UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq)
857 static inline void gen_unary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
858 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm)
859 {
860 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
861 if (!s->vex_l) {
862 xmm(OP_PTR0, OP_PTR1, imm);
863 } else {
864 ymm(OP_PTR0, OP_PTR1, imm);
865 }
866 }
868 #define UNARY_IMM_SSE(uname, lname) \
869 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
870 { \
871 gen_unary_imm_sse(s, env, decode, \
872 gen_helper_##lname##_xmm, \
873 gen_helper_##lname##_ymm); \
874 }
876 UNARY_IMM_SSE(PSHUFD, pshufd)
877 UNARY_IMM_SSE(PSHUFHW, pshufhw)
878 UNARY_IMM_SSE(PSHUFLW, pshuflw)
879 #define gen_helper_vpermq_xmm NULL
880 UNARY_IMM_SSE(VPERMQ, vpermq)
881 UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm)
882 UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm)
884 static inline void gen_unary_imm_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
885 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm)
886 {
887 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
888 if (!s->vex_l) {
889 xmm(cpu_env, OP_PTR0, OP_PTR1, imm);
890 } else {
891 ymm(cpu_env, OP_PTR0, OP_PTR1, imm);
892 }
893 }
895 #define UNARY_IMM_FP_SSE(uname, lname) \
896 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
897 { \
898 gen_unary_imm_fp_sse(s, env, decode, \
899 gen_helper_##lname##_xmm, \
900 gen_helper_##lname##_ymm); \
901 }
903 UNARY_IMM_FP_SSE(VROUNDPS, roundps)
904 UNARY_IMM_FP_SSE(VROUNDPD, roundpd)
906 static inline void gen_vexw_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
907 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm,
908 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm)
909 {
910 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm;
911 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm;
912 SSEFunc_0_eppp fn = s->vex_w ? q : d;
913 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
914 }
916 /* VEX.W affects whether to operate on 32- or 64-bit elements. */
917 #define VEXW_AVX(uname, lname) \
918 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
919 { \
920 gen_vexw_avx(s, env, decode, \
921 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \
922 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \
923 }
924 VEXW_AVX(VPSLLV, vpsllv)
925 VEXW_AVX(VPSRLV, vpsrlv)
926 VEXW_AVX(VPSRAV, vpsrav)
927 VEXW_AVX(VPMASKMOV, vpmaskmov)
929 /* Same as above, but with extra arguments to the helper. */
930 static inline void gen_vsib_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
931 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm,
932 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm)
933 {
934 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm;
935 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm;
936 SSEFunc_0_epppti fn = s->vex_w ? q : d;
937 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale);
938 TCGv_ptr index = tcg_temp_new_ptr();
940 /* Pass third input as (index, base, scale) */
941 tcg_gen_addi_ptr(index, cpu_env, ZMM_OFFSET(decode->mem.index));
942 fn(cpu_env, OP_PTR0, OP_PTR1, index, s->A0, scale);
944 /*
945 * There are two output operands, so zero OP1's high 128 bits
946 * in the VEX.128 case.
947 */
948 if (!s->vex_l) {
949 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1);
950 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0);
951 }
952 tcg_temp_free_ptr(index);
953 }
954 #define VSIB_AVX(uname, lname) \
955 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
956 { \
957 gen_vsib_avx(s, env, decode, \
958 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \
959 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \
960 }
961 VSIB_AVX(VPGATHERD, vpgatherd)
962 VSIB_AVX(VPGATHERQ, vpgatherq)
964 static void gen_ADCOX(DisasContext *s, CPUX86State *env, MemOp ot, int cc_op)
965 {
966 TCGv carry_in = NULL;
967 TCGv carry_out = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2);
968 TCGv zero;
970 if (cc_op == s->cc_op || s->cc_op == CC_OP_ADCOX) {
971 /* Re-use the carry-out from a previous round. */
972 carry_in = carry_out;
973 cc_op = s->cc_op;
974 } else if (s->cc_op == CC_OP_ADCX || s->cc_op == CC_OP_ADOX) {
975 /* Merge with the carry-out from the opposite instruction. */
976 cc_op = CC_OP_ADCOX;
977 }
979 /* If we don't have a carry-in, get it out of EFLAGS. */
980 if (!carry_in) {
981 if (s->cc_op != CC_OP_ADCX && s->cc_op != CC_OP_ADOX) {
982 gen_compute_eflags(s);
983 }
984 carry_in = s->tmp0;
985 tcg_gen_extract_tl(carry_in, cpu_cc_src,
986 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1);
987 }
989 switch (ot) {
990 #ifdef TARGET_X86_64
991 case MO_32:
992 /* If TL is 64-bit just do everything in 64-bit arithmetic. */
993 tcg_gen_add_i64(s->T0, s->T0, s->T1);
994 tcg_gen_add_i64(s->T0, s->T0, carry_in);
995 tcg_gen_shri_i64(carry_out, s->T0, 32);
996 break;
997 #endif
998 default:
999 zero = tcg_constant_tl(0);
1000 tcg_gen_add2_tl(s->T0, carry_out, s->T0, zero, carry_in, zero);
1001 tcg_gen_add2_tl(s->T0, carry_out, s->T0, carry_out, s->T1, zero);
1002 break;
1003 }
1004 set_cc_op(s, cc_op);
1005 }
1007 static void gen_ADCX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1008 {
1009 gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADCX);
1010 }
1012 static void gen_ADOX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1013 {
1014 gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADOX);
1015 }
1017 static void gen_ANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1018 {
1019 MemOp ot = decode->op[0].ot;
1021 tcg_gen_andc_tl(s->T0, s->T1, s->T0);
1022 gen_op_update1_cc(s);
1023 set_cc_op(s, CC_OP_LOGICB + ot);
1024 }
1026 static void gen_BEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1027 {
1028 MemOp ot = decode->op[0].ot;
1029 TCGv bound, zero;
1031 /*
1032 * Extract START, and shift the operand.
1033 * Shifts larger than operand size get zeros.
1034 */
1035 tcg_gen_ext8u_tl(s->A0, s->T1);
1036 tcg_gen_shr_tl(s->T0, s->T0, s->A0);
1038 bound = tcg_constant_tl(ot == MO_64 ? 63 : 31);
1039 zero = tcg_constant_tl(0);
1040 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero);
1042 /*
1043 * Extract the LEN into a mask. Lengths larger than
1044 * operand size get all ones.
1045 */
1046 tcg_gen_extract_tl(s->A0, s->T1, 8, 8);
1047 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->A0, bound, s->A0, bound);
1049 tcg_gen_movi_tl(s->T1, 1);
1050 tcg_gen_shl_tl(s->T1, s->T1, s->A0);
1051 tcg_gen_subi_tl(s->T1, s->T1, 1);
1052 tcg_gen_and_tl(s->T0, s->T0, s->T1);
1054 gen_op_update1_cc(s);
1055 set_cc_op(s, CC_OP_LOGICB + ot);
1056 }
1058 static void gen_BLSI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1059 {
1060 MemOp ot = decode->op[0].ot;
1062 tcg_gen_neg_tl(s->T1, s->T0);
1063 tcg_gen_and_tl(s->T0, s->T0, s->T1);
1064 tcg_gen_mov_tl(cpu_cc_dst, s->T0);
1065 set_cc_op(s, CC_OP_BMILGB + ot);
1066 }
1068 static void gen_BLSMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1069 {
1070 MemOp ot = decode->op[0].ot;
1072 tcg_gen_subi_tl(s->T1, s->T0, 1);
1073 tcg_gen_xor_tl(s->T0, s->T0, s->T1);
1074 tcg_gen_mov_tl(cpu_cc_dst, s->T0);
1075 set_cc_op(s, CC_OP_BMILGB + ot);
1076 }
1078 static void gen_BLSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1079 {
1080 MemOp ot = decode->op[0].ot;
1082 tcg_gen_subi_tl(s->T1, s->T0, 1);
1083 tcg_gen_and_tl(s->T0, s->T0, s->T1);
1084 tcg_gen_mov_tl(cpu_cc_dst, s->T0);
1085 set_cc_op(s, CC_OP_BMILGB + ot);
1086 }
1088 static void gen_BZHI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1089 {
1090 MemOp ot = decode->op[0].ot;
1091 TCGv bound;
1093 tcg_gen_ext8u_tl(s->T1, cpu_regs[s->vex_v]);
1094 bound = tcg_constant_tl(ot == MO_64 ? 63 : 31);
1096 /*
1097 * Note that since we're using BMILG (in order to get O
1098 * cleared) we need to store the inverse into C.
1099 */
1100 tcg_gen_setcond_tl(TCG_COND_LT, cpu_cc_src, s->T1, bound);
1101 tcg_gen_movcond_tl(TCG_COND_GT, s->T1, s->T1, bound, bound, s->T1);
1103 tcg_gen_movi_tl(s->A0, -1);
1104 tcg_gen_shl_tl(s->A0, s->A0, s->T1);
1105 tcg_gen_andc_tl(s->T0, s->T0, s->A0);
1107 gen_op_update1_cc(s);
1108 set_cc_op(s, CC_OP_BMILGB + ot);
1109 }
1111 static void gen_CRC32(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1112 {
1113 MemOp ot = decode->op[2].ot;
1115 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0);
1116 gen_helper_crc32(s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(8 << ot));
1117 }
1119 static void gen_CVTPI2Px(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1120 {
1121 gen_helper_enter_mmx(cpu_env);
1122 if (s->prefix & PREFIX_DATA) {
1123 gen_helper_cvtpi2pd(cpu_env, OP_PTR0, OP_PTR2);
1124 } else {
1125 gen_helper_cvtpi2ps(cpu_env, OP_PTR0, OP_PTR2);
1126 }
1127 }
1129 static void gen_CVTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1130 {
1131 gen_helper_enter_mmx(cpu_env);
1132 if (s->prefix & PREFIX_DATA) {
1133 gen_helper_cvtpd2pi(cpu_env, OP_PTR0, OP_PTR2);
1134 } else {
1135 gen_helper_cvtps2pi(cpu_env, OP_PTR0, OP_PTR2);
1136 }
1137 }
1139 static void gen_CVTTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1140 {
1141 gen_helper_enter_mmx(cpu_env);
1142 if (s->prefix & PREFIX_DATA) {
1143 gen_helper_cvttpd2pi(cpu_env, OP_PTR0, OP_PTR2);
1144 } else {
1145 gen_helper_cvttps2pi(cpu_env, OP_PTR0, OP_PTR2);
1146 }
1147 }
1149 static void gen_EMMS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1150 {
1151 gen_helper_emms(cpu_env);
1152 }
1154 static void gen_EXTRQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1155 {
1156 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63);
1157 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63);
1159 gen_helper_extrq_i(cpu_env, OP_PTR0, index, length);
1160 }
1162 static void gen_EXTRQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1163 {
1164 gen_helper_extrq_r(cpu_env, OP_PTR0, OP_PTR2);
1165 }
1167 static void gen_INSERTQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1168 {
1169 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63);
1170 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63);
1172 gen_helper_insertq_i(cpu_env, OP_PTR0, OP_PTR1, index, length);
1173 }
1175 static void gen_INSERTQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1176 {
1177 gen_helper_insertq_r(cpu_env, OP_PTR0, OP_PTR2);
1178 }
1180 static void gen_LDMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1181 {
1182 if (s->vex_l) {
1183 gen_illegal_opcode(s);
1184 return;
1185 }
1186 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T1);
1187 gen_helper_ldmxcsr(cpu_env, s->tmp2_i32);
1188 }
1190 static void gen_MASKMOV(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1191 {
1192 tcg_gen_mov_tl(s->A0, cpu_regs[R_EDI]);
1193 gen_extu(s->aflag, s->A0);
1194 gen_add_A0_ds_seg(s);
1196 if (s->prefix & PREFIX_DATA) {
1197 gen_helper_maskmov_xmm(cpu_env, OP_PTR1, OP_PTR2, s->A0);
1198 } else {
1199 gen_helper_maskmov_mmx(cpu_env, OP_PTR1, OP_PTR2, s->A0);
1200 }
1201 }
1203 static void gen_MOVBE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1204 {
1205 MemOp ot = decode->op[0].ot;
1207 /* M operand type does not load/store */
1208 if (decode->e.op0 == X86_TYPE_M) {
1209 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE);
1210 } else {
1211 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE);
1212 }
1213 }
1215 static void gen_MOVD_from(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1216 {
1217 MemOp ot = decode->op[2].ot;
1219 switch (ot) {
1220 case MO_32:
1221 #ifdef TARGET_X86_64
1222 tcg_gen_ld32u_tl(s->T0, cpu_env, decode->op[2].offset);
1223 break;
1224 case MO_64:
1225 #endif
1226 tcg_gen_ld_tl(s->T0, cpu_env, decode->op[2].offset);
1227 break;
1228 default:
1229 abort();
1230 }
1231 }
1233 static void gen_MOVD_to(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1234 {
1235 MemOp ot = decode->op[2].ot;
1236 int vec_len = vector_len(s, decode);
1237 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0);
1239 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1241 switch (ot) {
1242 case MO_32:
1243 #ifdef TARGET_X86_64
1244 tcg_gen_st32_tl(s->T1, cpu_env, lo_ofs);
1245 break;
1246 case MO_64:
1247 #endif
1248 tcg_gen_st_tl(s->T1, cpu_env, lo_ofs);
1249 break;
1250 default:
1251 g_assert_not_reached();
1252 }
1253 }
1255 static void gen_MOVDQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1256 {
1257 gen_store_sse(s, decode, decode->op[2].offset);
1258 }
1260 static void gen_MOVMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1261 {
1262 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn;
1263 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm;
1264 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm;
1265 fn = s->prefix & PREFIX_DATA ? pd : ps;
1266 fn(s->tmp2_i32, cpu_env, OP_PTR2);
1267 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32);
1268 }
1270 static void gen_MOVQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1271 {
1272 int vec_len = vector_len(s, decode);
1273 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0);
1275 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset);
1276 if (decode->op[0].has_ea) {
1277 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
1278 } else {
1279 /*
1280 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would
1281 * seem to work, but it does not on big-endian platforms; the cleared parts
1282 * are always at higher addresses, but cross-endian emulation inverts the
1283 * byte order so that the cleared parts need to be at *lower* addresses.
1284 * Because oprsz is 8, we see this here even for SSE; but more in general,
1285 * it disqualifies using oprsz < maxsz to emulate VEX128.
1286 */
1287 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1288 tcg_gen_st_i64(s->tmp1_i64, cpu_env, lo_ofs);
1289 }
1290 }
1292 static void gen_MOVq_dq(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1293 {
1294 gen_helper_enter_mmx(cpu_env);
1295 /* Otherwise the same as any other movq. */
1296 return gen_MOVQ(s, env, decode);
1297 }
1299 static void gen_MULX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1300 {
1301 MemOp ot = decode->op[0].ot;
1303 /* low part of result in VEX.vvvv, high in MODRM */
1304 switch (ot) {
1305 default:
1306 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0);
1307 tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1);
1308 tcg_gen_mulu2_i32(s->tmp2_i32, s->tmp3_i32,
1309 s->tmp2_i32, s->tmp3_i32);
1310 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], s->tmp2_i32);
1311 tcg_gen_extu_i32_tl(s->T0, s->tmp3_i32);
1312 break;
1313 #ifdef TARGET_X86_64
1314 case MO_64:
1315 tcg_gen_mulu2_i64(cpu_regs[s->vex_v], s->T0, s->T0, s->T1);
1316 break;
1317 #endif
1318 }
1320 }
1322 static void gen_PALIGNR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1323 {
1324 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1325 if (!(s->prefix & PREFIX_DATA)) {
1326 gen_helper_palignr_mmx(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
1327 } else if (!s->vex_l) {
1328 gen_helper_palignr_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
1329 } else {
1330 gen_helper_palignr_ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
1331 }
1332 }
1334 static void gen_PANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1335 {
1336 int vec_len = vector_len(s, decode);
1338 /* Careful, operand order is reversed! */
1339 tcg_gen_gvec_andc(MO_64,
1340 decode->op[0].offset, decode->op[2].offset,
1341 decode->op[1].offset, vec_len, vec_len);
1342 }
1344 static void gen_PCMPESTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1345 {
1346 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1347 gen_helper_pcmpestri_xmm(cpu_env, OP_PTR1, OP_PTR2, imm);
1348 set_cc_op(s, CC_OP_EFLAGS);
1349 }
1351 static void gen_PCMPESTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1352 {
1353 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1354 gen_helper_pcmpestrm_xmm(cpu_env, OP_PTR1, OP_PTR2, imm);
1355 set_cc_op(s, CC_OP_EFLAGS);
1356 if ((s->prefix & PREFIX_VEX) && !s->vex_l) {
1357 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)),
1358 16, 16, 0);
1359 }
1360 }
1362 static void gen_PCMPISTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1363 {
1364 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1365 gen_helper_pcmpistri_xmm(cpu_env, OP_PTR1, OP_PTR2, imm);
1366 set_cc_op(s, CC_OP_EFLAGS);
1367 }
1369 static void gen_PCMPISTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1370 {
1371 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1372 gen_helper_pcmpistrm_xmm(cpu_env, OP_PTR1, OP_PTR2, imm);
1373 set_cc_op(s, CC_OP_EFLAGS);
1374 if ((s->prefix & PREFIX_VEX) && !s->vex_l) {
1375 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)),
1376 16, 16, 0);
1377 }
1378 }
1380 static void gen_PDEP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1381 {
1382 MemOp ot = decode->op[1].ot;
1383 if (ot < MO_64) {
1384 tcg_gen_ext32u_tl(s->T0, s->T0);
1385 }
1386 gen_helper_pdep(s->T0, s->T0, s->T1);
1387 }
1389 static void gen_PEXT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1390 {
1391 MemOp ot = decode->op[1].ot;
1392 if (ot < MO_64) {
1393 tcg_gen_ext32u_tl(s->T0, s->T0);
1394 }
1395 gen_helper_pext(s->T0, s->T0, s->T1);
1396 }
1398 static inline void gen_pextr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot)
1399 {
1400 int vec_len = vector_len(s, decode);
1401 int mask = (vec_len >> ot) - 1;
1402 int val = decode->immediate & mask;
1404 switch (ot) {
1405 case MO_8:
1406 tcg_gen_ld8u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val));
1407 break;
1408 case MO_16:
1409 tcg_gen_ld16u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val));
1410 break;
1411 case MO_32:
1412 #ifdef TARGET_X86_64
1413 tcg_gen_ld32u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val));
1414 break;
1415 case MO_64:
1416 #endif
1417 tcg_gen_ld_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val));
1418 break;
1419 default:
1420 abort();
1421 }
1422 }
1424 static void gen_PEXTRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1425 {
1426 gen_pextr(s, env, decode, MO_8);
1427 }
1429 static void gen_PEXTRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1430 {
1431 gen_pextr(s, env, decode, MO_16);
1432 }
1434 static void gen_PEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1435 {
1436 MemOp ot = decode->op[0].ot;
1437 gen_pextr(s, env, decode, ot);
1438 }
1440 static inline void gen_pinsr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot)
1441 {
1442 int vec_len = vector_len(s, decode);
1443 int mask = (vec_len >> ot) - 1;
1444 int val = decode->immediate & mask;
1446 if (decode->op[1].offset != decode->op[0].offset) {
1447 assert(vec_len == 16);
1448 gen_store_sse(s, decode, decode->op[1].offset);
1449 }
1451 switch (ot) {
1452 case MO_8:
1453 tcg_gen_st8_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val));
1454 break;
1455 case MO_16:
1456 tcg_gen_st16_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val));
1457 break;
1458 case MO_32:
1459 #ifdef TARGET_X86_64
1460 tcg_gen_st32_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val));
1461 break;
1462 case MO_64:
1463 #endif
1464 tcg_gen_st_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val));
1465 break;
1466 default:
1467 abort();
1468 }
1469 }
1471 static void gen_PINSRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1472 {
1473 gen_pinsr(s, env, decode, MO_8);
1474 }
1476 static void gen_PINSRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1477 {
1478 gen_pinsr(s, env, decode, MO_16);
1479 }
1481 static void gen_PINSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1482 {
1483 gen_pinsr(s, env, decode, decode->op[2].ot);
1484 }
1486 static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s)
1487 {
1488 TCGv_i64 t = tcg_temp_new_i64();
1490 tcg_gen_andi_i64(d, s, 0x8080808080808080ull);
1492 /*
1493 * After each shift+or pair:
1494 * 0: a.......b.......c.......d.......e.......f.......g.......h.......
1495 * 7: ab......bc......cd......de......ef......fg......gh......h.......
1496 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h.......
1497 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h.......
1498 * The result is left in the high bits of the word.
1499 */
1500 tcg_gen_shli_i64(t, d, 7);
1501 tcg_gen_or_i64(d, d, t);
1502 tcg_gen_shli_i64(t, d, 14);
1503 tcg_gen_or_i64(d, d, t);
1504 tcg_gen_shli_i64(t, d, 28);
1505 tcg_gen_or_i64(d, d, t);
1506 }
1508 static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s)
1509 {
1510 TCGv_vec t = tcg_temp_new_vec_matching(d);
1511 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80);
1513 /* See above */
1514 tcg_gen_and_vec(vece, d, s, m);
1515 tcg_gen_shli_vec(vece, t, d, 7);
1516 tcg_gen_or_vec(vece, d, d, t);
1517 tcg_gen_shli_vec(vece, t, d, 14);
1518 tcg_gen_or_vec(vece, d, d, t);
1519 tcg_gen_shli_vec(vece, t, d, 28);
1520 tcg_gen_or_vec(vece, d, d, t);
1521 }
1523 #ifdef TARGET_X86_64
1524 #define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i64
1525 #else
1526 #define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i32
1527 #endif
1529 static void gen_PMOVMSKB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1530 {
1531 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
1532 static const GVecGen2 g = {
1533 .fni8 = gen_pmovmskb_i64,
1534 .fniv = gen_pmovmskb_vec,
1535 .opt_opc = vecop_list,
1536 .vece = MO_64,
1537 .prefer_i64 = TCG_TARGET_REG_BITS == 64
1538 };
1539 MemOp ot = decode->op[2].ot;
1540 int vec_len = vector_len(s, decode);
1541 TCGv t = tcg_temp_new();
1543 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset,
1544 vec_len, vec_len, &g);
1545 tcg_gen_ld8u_tl(s->T0, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1)));
1546 while (vec_len > 8) {
1547 vec_len -= 8;
1548 if (TCG_TARGET_HAS_extract2_tl) {
1549 /*
1550 * Load the next byte of the result into the high byte of T.
1551 * TCG does a similar expansion of deposit to shl+extract2; by
1552 * loading the whole word, the shift left is avoided.
1553 */
1554 #ifdef TARGET_X86_64
1555 tcg_gen_ld_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8)));
1556 #else
1557 tcg_gen_ld_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4)));
1558 #endif
1560 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8);
1561 } else {
1562 /*
1563 * The _previous_ value is deposited into bits 8 and higher of t. Because
1564 * those bits are known to be zero after ld8u, this becomes a shift+or
1565 * if deposit is not available.
1566 */
1567 tcg_gen_ld8u_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1)));
1568 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8);
1569 }
1570 }
1571 tcg_temp_free(t);
1572 }
1574 static void gen_PSHUFW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1575 {
1576 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1577 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm);
1578 }
1580 static void gen_PSRLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1581 {
1582 int vec_len = vector_len(s, decode);
1584 if (decode->immediate >= 16) {
1585 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1586 } else {
1587 tcg_gen_gvec_shri(MO_16,
1588 decode->op[0].offset, decode->op[1].offset,
1589 decode->immediate, vec_len, vec_len);
1590 }
1591 }
1593 static void gen_PSLLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1594 {
1595 int vec_len = vector_len(s, decode);
1597 if (decode->immediate >= 16) {
1598 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1599 } else {
1600 tcg_gen_gvec_shli(MO_16,
1601 decode->op[0].offset, decode->op[1].offset,
1602 decode->immediate, vec_len, vec_len);
1603 }
1604 }
1606 static void gen_PSRAW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1607 {
1608 int vec_len = vector_len(s, decode);
1610 if (decode->immediate >= 16) {
1611 decode->immediate = 15;
1612 }
1613 tcg_gen_gvec_sari(MO_16,
1614 decode->op[0].offset, decode->op[1].offset,
1615 decode->immediate, vec_len, vec_len);
1616 }
1618 static void gen_PSRLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1619 {
1620 int vec_len = vector_len(s, decode);
1622 if (decode->immediate >= 32) {
1623 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1624 } else {
1625 tcg_gen_gvec_shri(MO_32,
1626 decode->op[0].offset, decode->op[1].offset,
1627 decode->immediate, vec_len, vec_len);
1628 }
1629 }
1631 static void gen_PSLLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1632 {
1633 int vec_len = vector_len(s, decode);
1635 if (decode->immediate >= 32) {
1636 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1637 } else {
1638 tcg_gen_gvec_shli(MO_32,
1639 decode->op[0].offset, decode->op[1].offset,
1640 decode->immediate, vec_len, vec_len);
1641 }
1642 }
1644 static void gen_PSRAD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1645 {
1646 int vec_len = vector_len(s, decode);
1648 if (decode->immediate >= 32) {
1649 decode->immediate = 31;
1650 }
1651 tcg_gen_gvec_sari(MO_32,
1652 decode->op[0].offset, decode->op[1].offset,
1653 decode->immediate, vec_len, vec_len);
1654 }
1656 static void gen_PSRLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1657 {
1658 int vec_len = vector_len(s, decode);
1660 if (decode->immediate >= 64) {
1661 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1662 } else {
1663 tcg_gen_gvec_shri(MO_64,
1664 decode->op[0].offset, decode->op[1].offset,
1665 decode->immediate, vec_len, vec_len);
1666 }
1667 }
1669 static void gen_PSLLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1670 {
1671 int vec_len = vector_len(s, decode);
1673 if (decode->immediate >= 64) {
1674 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1675 } else {
1676 tcg_gen_gvec_shli(MO_64,
1677 decode->op[0].offset, decode->op[1].offset,
1678 decode->immediate, vec_len, vec_len);
1679 }
1680 }
1682 static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len)
1683 {
1684 MemOp ot = vec_len == 16 ? MO_128 : MO_256;
1685 TCGv_i32 imm_v = tcg_constant8u_i32(imm);
1686 TCGv_ptr ptr = tcg_temp_new_ptr();
1688 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot),
1689 vec_len, vec_len, 0);
1691 tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_t0));
1692 tcg_gen_st_i32(imm_v, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0)));
1693 return ptr;
1694 }
1696 static void gen_PSRLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1697 {
1698 int vec_len = vector_len(s, decode);
1699 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len);
1701 if (s->vex_l) {
1702 gen_helper_psrldq_ymm(cpu_env, OP_PTR0, OP_PTR1, imm_vec);
1703 } else {
1704 gen_helper_psrldq_xmm(cpu_env, OP_PTR0, OP_PTR1, imm_vec);
1705 }
1706 tcg_temp_free_ptr(imm_vec);
1707 }
1709 static void gen_PSLLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1710 {
1711 int vec_len = vector_len(s, decode);
1712 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len);
1714 if (s->vex_l) {
1715 gen_helper_pslldq_ymm(cpu_env, OP_PTR0, OP_PTR1, imm_vec);
1716 } else {
1717 gen_helper_pslldq_xmm(cpu_env, OP_PTR0, OP_PTR1, imm_vec);
1718 }
1719 tcg_temp_free_ptr(imm_vec);
1720 }
1722 static void gen_RORX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1723 {
1724 MemOp ot = decode->op[0].ot;
1725 int b = decode->immediate;
1727 if (ot == MO_64) {
1728 tcg_gen_rotri_tl(s->T0, s->T0, b & 63);
1729 } else {
1730 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0);
1731 tcg_gen_rotri_i32(s->tmp2_i32, s->tmp2_i32, b & 31);
1732 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32);
1733 }
1734 }
1736 static void gen_SARX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1737 {
1738 MemOp ot = decode->op[0].ot;
1739 int mask;
1741 mask = ot == MO_64 ? 63 : 31;
1742 tcg_gen_andi_tl(s->T1, s->T1, mask);
1743 if (ot != MO_64) {
1744 tcg_gen_ext32s_tl(s->T0, s->T0);
1745 }
1746 tcg_gen_sar_tl(s->T0, s->T0, s->T1);
1747 }
1749 static void gen_SHLX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1750 {
1751 MemOp ot = decode->op[0].ot;
1752 int mask;
1754 mask = ot == MO_64 ? 63 : 31;
1755 tcg_gen_andi_tl(s->T1, s->T1, mask);
1756 tcg_gen_shl_tl(s->T0, s->T0, s->T1);
1757 }
1759 static void gen_SHRX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1760 {
1761 MemOp ot = decode->op[0].ot;
1762 int mask;
1764 mask = ot == MO_64 ? 63 : 31;
1765 tcg_gen_andi_tl(s->T1, s->T1, mask);
1766 if (ot != MO_64) {
1767 tcg_gen_ext32u_tl(s->T0, s->T0);
1768 }
1769 tcg_gen_shr_tl(s->T0, s->T0, s->T1);
1770 }
1772 static void gen_VAESKEYGEN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1773 {
1774 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1775 assert(!s->vex_l);
1776 gen_helper_aeskeygenassist_xmm(cpu_env, OP_PTR0, OP_PTR1, imm);
1777 }
1779 static void gen_STMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1780 {
1781 if (s->vex_l) {
1782 gen_illegal_opcode(s);
1783 return;
1784 }
1785 gen_helper_update_mxcsr(cpu_env);
1786 tcg_gen_ld32u_tl(s->T0, cpu_env, offsetof(CPUX86State, mxcsr));
1787 }
1789 static void gen_VAESIMC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1790 {
1791 assert(!s->vex_l);
1792 gen_helper_aesimc_xmm(cpu_env, OP_PTR0, OP_PTR2);
1793 }
1795 /*
1796 * 00 = v*ps Vps, Hps, Wpd
1797 * 66 = v*pd Vpd, Hpd, Wps
1798 * f3 = v*ss Vss, Hss, Wps
1799 * f2 = v*sd Vsd, Hsd, Wps
1800 */
1801 #define SSE_CMP(x) { \
1802 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \
1803 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \
1804 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm}
1805 static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = {
1806 SSE_CMP(cmpeq),
1807 SSE_CMP(cmplt),
1808 SSE_CMP(cmple),
1809 SSE_CMP(cmpunord),
1810 SSE_CMP(cmpneq),
1811 SSE_CMP(cmpnlt),
1812 SSE_CMP(cmpnle),
1813 SSE_CMP(cmpord),
1815 SSE_CMP(cmpequ),
1816 SSE_CMP(cmpnge),
1817 SSE_CMP(cmpngt),
1818 SSE_CMP(cmpfalse),
1819 SSE_CMP(cmpnequ),
1820 SSE_CMP(cmpge),
1821 SSE_CMP(cmpgt),
1822 SSE_CMP(cmptrue),
1824 SSE_CMP(cmpeqs),
1825 SSE_CMP(cmpltq),
1826 SSE_CMP(cmpleq),
1827 SSE_CMP(cmpunords),
1828 SSE_CMP(cmpneqq),
1829 SSE_CMP(cmpnltq),
1830 SSE_CMP(cmpnleq),
1831 SSE_CMP(cmpords),
1833 SSE_CMP(cmpequs),
1834 SSE_CMP(cmpngeq),
1835 SSE_CMP(cmpngtq),
1836 SSE_CMP(cmpfalses),
1837 SSE_CMP(cmpnequs),
1838 SSE_CMP(cmpgeq),
1839 SSE_CMP(cmpgtq),
1840 SSE_CMP(cmptrues),
1841 };
1842 #undef SSE_CMP
1844 static void gen_VCMP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1845 {
1846 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7);
1847 int b =
1848 s->prefix & PREFIX_REPZ ? 2 /* ss */ :
1849 s->prefix & PREFIX_REPNZ ? 3 /* sd */ :
1850 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2);
1852 gen_helper_cmp_funcs[index][b](cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
1853 }
1855 static void gen_VCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1856 {
1857 SSEFunc_0_epp fn;
1858 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss;
1859 fn(cpu_env, OP_PTR1, OP_PTR2);
1860 set_cc_op(s, CC_OP_EFLAGS);
1861 }
1863 static void gen_VCVTfp2fp(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1864 {
1865 gen_unary_fp_sse(s, env, decode,
1866 gen_helper_cvtpd2ps_xmm, gen_helper_cvtps2pd_xmm,
1867 gen_helper_cvtpd2ps_ymm, gen_helper_cvtps2pd_ymm,
1868 gen_helper_cvtsd2ss, gen_helper_cvtss2sd);
1869 }
1871 static void gen_VCVTSI2Sx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1872 {
1873 int vec_len = vector_len(s, decode);
1874 TCGv_i32 in;
1876 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
1878 #ifdef TARGET_X86_64
1879 MemOp ot = decode->op[2].ot;
1880 if (ot == MO_64) {
1881 if (s->prefix & PREFIX_REPNZ) {
1882 gen_helper_cvtsq2sd(cpu_env, OP_PTR0, s->T1);
1883 } else {
1884 gen_helper_cvtsq2ss(cpu_env, OP_PTR0, s->T1);
1885 }
1886 return;
1887 }
1888 in = s->tmp2_i32;
1889 tcg_gen_trunc_tl_i32(in, s->T1);
1890 #else
1891 in = s->T1;
1892 #endif
1894 if (s->prefix & PREFIX_REPNZ) {
1895 gen_helper_cvtsi2sd(cpu_env, OP_PTR0, in);
1896 } else {
1897 gen_helper_cvtsi2ss(cpu_env, OP_PTR0, in);
1898 }
1899 }
1901 static inline void gen_VCVTtSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
1902 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq,
1903 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq)
1904 {
1905 TCGv_i32 out;
1907 #ifdef TARGET_X86_64
1908 MemOp ot = decode->op[0].ot;
1909 if (ot == MO_64) {
1910 if (s->prefix & PREFIX_REPNZ) {
1911 sd2sq(s->T0, cpu_env, OP_PTR2);
1912 } else {
1913 ss2sq(s->T0, cpu_env, OP_PTR2);
1914 }
1915 return;
1916 }
1918 out = s->tmp2_i32;
1919 #else
1920 out = s->T0;
1921 #endif
1922 if (s->prefix & PREFIX_REPNZ) {
1923 sd2si(out, cpu_env, OP_PTR2);
1924 } else {
1925 ss2si(out, cpu_env, OP_PTR2);
1926 }
1927 #ifdef TARGET_X86_64
1928 tcg_gen_extu_i32_tl(s->T0, out);
1929 #endif
1930 }
1932 #ifndef TARGET_X86_64
1933 #define gen_helper_cvtss2sq NULL
1934 #define gen_helper_cvtsd2sq NULL
1935 #define gen_helper_cvttss2sq NULL
1936 #define gen_helper_cvttsd2sq NULL
1937 #endif
1939 static void gen_VCVTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1940 {
1941 gen_VCVTtSx2SI(s, env, decode,
1942 gen_helper_cvtss2si, gen_helper_cvtss2sq,
1943 gen_helper_cvtsd2si, gen_helper_cvtsd2sq);
1944 }
1946 static void gen_VCVTTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1947 {
1948 gen_VCVTtSx2SI(s, env, decode,
1949 gen_helper_cvttss2si, gen_helper_cvttss2sq,
1950 gen_helper_cvttsd2si, gen_helper_cvttsd2sq);
1951 }
1953 static void gen_VEXTRACTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1954 {
1955 int mask = decode->immediate & 1;
1956 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask);
1957 if (decode->op[0].has_ea) {
1958 /* VEX-only instruction, no alignment requirements. */
1959 gen_sto_env_A0(s, src_ofs, false);
1960 } else {
1961 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16);
1962 }
1963 }
1965 static void gen_VEXTRACTPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1966 {
1967 gen_pextr(s, env, decode, MO_32);
1968 }
1970 static void gen_vinsertps(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
1971 {
1972 int val = decode->immediate;
1973 int dest_word = (val >> 4) & 3;
1974 int new_mask = (val & 15) | (1 << dest_word);
1975 int vec_len = 16;
1977 assert(!s->vex_l);
1979 if (new_mask == 15) {
1980 /* All zeroes except possibly for the inserted element */
1981 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
1982 } else if (decode->op[1].offset != decode->op[0].offset) {
1983 gen_store_sse(s, decode, decode->op[1].offset);
1984 }
1986 if (new_mask != (val & 15)) {
1987 tcg_gen_st_i32(s->tmp2_i32, cpu_env,
1988 vector_elem_offset(&decode->op[0], MO_32, dest_word));
1989 }
1991 if (new_mask != 15) {
1992 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */
1993 int i;
1994 for (i = 0; i < 4; i++) {
1995 if ((val >> i) & 1) {
1996 tcg_gen_st_i32(zero, cpu_env,
1997 vector_elem_offset(&decode->op[0], MO_32, i));
1998 }
1999 }
2000 }
2001 }
2003 static void gen_VINSERTPS_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2004 {
2005 int val = decode->immediate;
2006 tcg_gen_ld_i32(s->tmp2_i32, cpu_env,
2007 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3));
2008 gen_vinsertps(s, env, decode);
2009 }
2011 static void gen_VINSERTPS_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2012 {
2013 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL);
2014 gen_vinsertps(s, env, decode);
2015 }
2017 static void gen_VINSERTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2018 {
2019 int mask = decode->immediate & 1;
2020 tcg_gen_gvec_mov(MO_64,
2021 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)),
2022 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16);
2023 tcg_gen_gvec_mov(MO_64,
2024 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)),
2025 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16);
2026 }
2028 static inline void gen_maskmov(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
2029 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm)
2030 {
2031 if (!s->vex_l) {
2032 xmm(cpu_env, OP_PTR2, OP_PTR1, s->A0);
2033 } else {
2034 ymm(cpu_env, OP_PTR2, OP_PTR1, s->A0);
2035 }
2036 }
2038 static void gen_VMASKMOVPD_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2039 {
2040 gen_maskmov(s, env, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm);
2041 }
2043 static void gen_VMASKMOVPS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2044 {
2045 gen_maskmov(s, env, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm);
2046 }
2048 static void gen_VMOVHPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2049 {
2050 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
2051 if (decode->op[0].offset != decode->op[1].offset) {
2052 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
2053 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
2054 }
2055 }
2057 static void gen_VMOVHPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2058 {
2059 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
2060 }
2062 static void gen_VMOVHPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2063 {
2064 if (decode->op[0].offset != decode->op[2].offset) {
2065 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
2066 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
2067 }
2068 if (decode->op[0].offset != decode->op[1].offset) {
2069 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
2070 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
2071 }
2072 }
2074 static void gen_VMOVHLPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2075 {
2076 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
2077 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
2078 if (decode->op[0].offset != decode->op[1].offset) {
2079 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1)));
2080 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
2081 }
2082 }
2084 static void gen_VMOVLHPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2085 {
2086 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset);
2087 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
2088 if (decode->op[0].offset != decode->op[1].offset) {
2089 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
2090 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
2091 }
2092 }
2094 /*
2095 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx.
2096 * Use a gvec move to move everything above the bottom 64 bits.
2097 */
2099 static void gen_VMOVLPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2100 {
2101 int vec_len = vector_len(s, decode);
2103 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0)));
2104 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
2105 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
2106 }
2108 static void gen_VMOVLPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2109 {
2110 int vec_len = vector_len(s, decode);
2112 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
2113 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
2114 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0)));
2115 }
2117 static void gen_VMOVLPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2118 {
2119 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0)));
2120 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
2121 }
2123 static void gen_VMOVSD_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2124 {
2125 TCGv_i64 zero = tcg_constant_i64(0);
2127 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
2128 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1)));
2129 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0)));
2130 }
2132 static void gen_VMOVSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2133 {
2134 int vec_len = vector_len(s, decode);
2136 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0)));
2137 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
2138 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0)));
2139 }
2141 static void gen_VMOVSS_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2142 {
2143 int vec_len = vector_len(s, decode);
2145 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL);
2146 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
2147 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0)));
2148 }
2150 static void gen_VMOVSS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2151 {
2152 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0)));
2153 tcg_gen_qemu_st_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL);
2154 }
2156 static void gen_VPMASKMOV_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2157 {
2158 if (s->vex_w) {
2159 gen_VMASKMOVPD_st(s, env, decode);
2160 } else {
2161 gen_VMASKMOVPS_st(s, env, decode);
2162 }
2163 }
2165 static void gen_VPERMD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2166 {
2167 assert(s->vex_l);
2168 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2);
2169 }
2171 static void gen_VPERM2x128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2172 {
2173 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
2174 assert(s->vex_l);
2175 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm);
2176 }
2178 static void gen_VPHMINPOSUW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2179 {
2180 assert(!s->vex_l);
2181 gen_helper_phminposuw_xmm(cpu_env, OP_PTR0, OP_PTR2);
2182 }
2184 static void gen_VROUNDSD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2185 {
2186 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
2187 assert(!s->vex_l);
2188 gen_helper_roundsd_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
2189 }
2191 static void gen_VROUNDSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2192 {
2193 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
2194 assert(!s->vex_l);
2195 gen_helper_roundss_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
2196 }
2198 static void gen_VSHUF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2199 {
2200 TCGv_i32 imm = tcg_constant_i32(decode->immediate);
2201 SSEFunc_0_pppi ps, pd, fn;
2202 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm;
2203 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm;
2204 fn = s->prefix & PREFIX_DATA ? pd : ps;
2205 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm);
2206 }
2208 static void gen_VUCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2209 {
2210 SSEFunc_0_epp fn;
2211 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss;
2212 fn(cpu_env, OP_PTR1, OP_PTR2);
2213 set_cc_op(s, CC_OP_EFLAGS);
2214 }
2216 static void gen_VZEROALL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2217 {
2218 TCGv_ptr ptr = tcg_temp_new_ptr();
2220 tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_t0));
2221 gen_helper_memset(ptr, ptr, tcg_constant_i32(0),
2222 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg)));
2223 tcg_temp_free_ptr(ptr);
2224 }
2226 static void gen_VZEROUPPER(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
2227 {
2228 int i;
2230 for (i = 0; i < CPU_NB_REGS; i++) {
2231 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1));
2232 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0);
2233 }
2234 }