2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 #include "qemu/osdep.h"
26 #include "exec/exec-all.h"
28 #include "tcg/tcg-op.h"
29 #include "tcg/tcg-mo.h"
30 #include "trace-tcg.h"
31 #include "exec/plugin-gen.h"
33 /* Reduce the number of ifdefs below. This assumes that all uses of
34 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
35 the compiler can eliminate. */
36 #if TCG_TARGET_REG_BITS == 64
37 extern TCGv_i32
TCGV_LOW_link_error(TCGv_i64
);
38 extern TCGv_i32
TCGV_HIGH_link_error(TCGv_i64
);
39 #define TCGV_LOW TCGV_LOW_link_error
40 #define TCGV_HIGH TCGV_HIGH_link_error
43 void tcg_gen_op1(TCGOpcode opc
, TCGArg a1
)
45 TCGOp
*op
= tcg_emit_op(opc
);
49 void tcg_gen_op2(TCGOpcode opc
, TCGArg a1
, TCGArg a2
)
51 TCGOp
*op
= tcg_emit_op(opc
);
56 void tcg_gen_op3(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
)
58 TCGOp
*op
= tcg_emit_op(opc
);
64 void tcg_gen_op4(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
, TCGArg a4
)
66 TCGOp
*op
= tcg_emit_op(opc
);
73 void tcg_gen_op5(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
,
76 TCGOp
*op
= tcg_emit_op(opc
);
84 void tcg_gen_op6(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
,
85 TCGArg a4
, TCGArg a5
, TCGArg a6
)
87 TCGOp
*op
= tcg_emit_op(opc
);
96 void tcg_gen_mb(TCGBar mb_type
)
98 if (tcg_ctx
->tb_cflags
& CF_PARALLEL
) {
99 tcg_gen_op1(INDEX_op_mb
, mb_type
);
105 void tcg_gen_movi_i32(TCGv_i32 ret
, int32_t arg
)
107 tcg_gen_mov_i32(ret
, tcg_constant_i32(arg
));
110 void tcg_gen_addi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
112 /* some cases can be optimized here */
114 tcg_gen_mov_i32(ret
, arg1
);
116 tcg_gen_add_i32(ret
, arg1
, tcg_constant_i32(arg2
));
120 void tcg_gen_subfi_i32(TCGv_i32 ret
, int32_t arg1
, TCGv_i32 arg2
)
122 if (arg1
== 0 && TCG_TARGET_HAS_neg_i32
) {
123 /* Don't recurse with tcg_gen_neg_i32. */
124 tcg_gen_op2_i32(INDEX_op_neg_i32
, ret
, arg2
);
126 tcg_gen_sub_i32(ret
, tcg_constant_i32(arg1
), arg2
);
130 void tcg_gen_subi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
132 /* some cases can be optimized here */
134 tcg_gen_mov_i32(ret
, arg1
);
136 tcg_gen_sub_i32(ret
, arg1
, tcg_constant_i32(arg2
));
140 void tcg_gen_andi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
142 /* Some cases can be optimized here. */
145 tcg_gen_movi_i32(ret
, 0);
148 tcg_gen_mov_i32(ret
, arg1
);
151 /* Don't recurse with tcg_gen_ext8u_i32. */
152 if (TCG_TARGET_HAS_ext8u_i32
) {
153 tcg_gen_op2_i32(INDEX_op_ext8u_i32
, ret
, arg1
);
158 if (TCG_TARGET_HAS_ext16u_i32
) {
159 tcg_gen_op2_i32(INDEX_op_ext16u_i32
, ret
, arg1
);
165 tcg_gen_and_i32(ret
, arg1
, tcg_constant_i32(arg2
));
168 void tcg_gen_ori_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
170 /* Some cases can be optimized here. */
172 tcg_gen_movi_i32(ret
, -1);
173 } else if (arg2
== 0) {
174 tcg_gen_mov_i32(ret
, arg1
);
176 tcg_gen_or_i32(ret
, arg1
, tcg_constant_i32(arg2
));
180 void tcg_gen_xori_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
182 /* Some cases can be optimized here. */
184 tcg_gen_mov_i32(ret
, arg1
);
185 } else if (arg2
== -1 && TCG_TARGET_HAS_not_i32
) {
186 /* Don't recurse with tcg_gen_not_i32. */
187 tcg_gen_op2_i32(INDEX_op_not_i32
, ret
, arg1
);
189 tcg_gen_xor_i32(ret
, arg1
, tcg_constant_i32(arg2
));
193 void tcg_gen_shli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
195 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
197 tcg_gen_mov_i32(ret
, arg1
);
199 tcg_gen_shl_i32(ret
, arg1
, tcg_constant_i32(arg2
));
203 void tcg_gen_shri_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
205 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
207 tcg_gen_mov_i32(ret
, arg1
);
209 tcg_gen_shr_i32(ret
, arg1
, tcg_constant_i32(arg2
));
213 void tcg_gen_sari_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
215 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
217 tcg_gen_mov_i32(ret
, arg1
);
219 tcg_gen_sar_i32(ret
, arg1
, tcg_constant_i32(arg2
));
223 void tcg_gen_brcond_i32(TCGCond cond
, TCGv_i32 arg1
, TCGv_i32 arg2
, TCGLabel
*l
)
225 if (cond
== TCG_COND_ALWAYS
) {
227 } else if (cond
!= TCG_COND_NEVER
) {
229 tcg_gen_op4ii_i32(INDEX_op_brcond_i32
, arg1
, arg2
, cond
, label_arg(l
));
233 void tcg_gen_brcondi_i32(TCGCond cond
, TCGv_i32 arg1
, int32_t arg2
, TCGLabel
*l
)
235 if (cond
== TCG_COND_ALWAYS
) {
237 } else if (cond
!= TCG_COND_NEVER
) {
238 tcg_gen_brcond_i32(cond
, arg1
, tcg_constant_i32(arg2
), l
);
242 void tcg_gen_setcond_i32(TCGCond cond
, TCGv_i32 ret
,
243 TCGv_i32 arg1
, TCGv_i32 arg2
)
245 if (cond
== TCG_COND_ALWAYS
) {
246 tcg_gen_movi_i32(ret
, 1);
247 } else if (cond
== TCG_COND_NEVER
) {
248 tcg_gen_movi_i32(ret
, 0);
250 tcg_gen_op4i_i32(INDEX_op_setcond_i32
, ret
, arg1
, arg2
, cond
);
254 void tcg_gen_setcondi_i32(TCGCond cond
, TCGv_i32 ret
,
255 TCGv_i32 arg1
, int32_t arg2
)
257 tcg_gen_setcond_i32(cond
, ret
, arg1
, tcg_constant_i32(arg2
));
260 void tcg_gen_muli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
263 tcg_gen_movi_i32(ret
, 0);
264 } else if (is_power_of_2(arg2
)) {
265 tcg_gen_shli_i32(ret
, arg1
, ctz32(arg2
));
267 tcg_gen_mul_i32(ret
, arg1
, tcg_constant_i32(arg2
));
271 void tcg_gen_div_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
273 if (TCG_TARGET_HAS_div_i32
) {
274 tcg_gen_op3_i32(INDEX_op_div_i32
, ret
, arg1
, arg2
);
275 } else if (TCG_TARGET_HAS_div2_i32
) {
276 TCGv_i32 t0
= tcg_temp_new_i32();
277 tcg_gen_sari_i32(t0
, arg1
, 31);
278 tcg_gen_op5_i32(INDEX_op_div2_i32
, ret
, t0
, arg1
, t0
, arg2
);
279 tcg_temp_free_i32(t0
);
281 gen_helper_div_i32(ret
, arg1
, arg2
);
285 void tcg_gen_rem_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
287 if (TCG_TARGET_HAS_rem_i32
) {
288 tcg_gen_op3_i32(INDEX_op_rem_i32
, ret
, arg1
, arg2
);
289 } else if (TCG_TARGET_HAS_div_i32
) {
290 TCGv_i32 t0
= tcg_temp_new_i32();
291 tcg_gen_op3_i32(INDEX_op_div_i32
, t0
, arg1
, arg2
);
292 tcg_gen_mul_i32(t0
, t0
, arg2
);
293 tcg_gen_sub_i32(ret
, arg1
, t0
);
294 tcg_temp_free_i32(t0
);
295 } else if (TCG_TARGET_HAS_div2_i32
) {
296 TCGv_i32 t0
= tcg_temp_new_i32();
297 tcg_gen_sari_i32(t0
, arg1
, 31);
298 tcg_gen_op5_i32(INDEX_op_div2_i32
, t0
, ret
, arg1
, t0
, arg2
);
299 tcg_temp_free_i32(t0
);
301 gen_helper_rem_i32(ret
, arg1
, arg2
);
305 void tcg_gen_divu_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
307 if (TCG_TARGET_HAS_div_i32
) {
308 tcg_gen_op3_i32(INDEX_op_divu_i32
, ret
, arg1
, arg2
);
309 } else if (TCG_TARGET_HAS_div2_i32
) {
310 TCGv_i32 t0
= tcg_temp_new_i32();
311 tcg_gen_movi_i32(t0
, 0);
312 tcg_gen_op5_i32(INDEX_op_divu2_i32
, ret
, t0
, arg1
, t0
, arg2
);
313 tcg_temp_free_i32(t0
);
315 gen_helper_divu_i32(ret
, arg1
, arg2
);
319 void tcg_gen_remu_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
321 if (TCG_TARGET_HAS_rem_i32
) {
322 tcg_gen_op3_i32(INDEX_op_remu_i32
, ret
, arg1
, arg2
);
323 } else if (TCG_TARGET_HAS_div_i32
) {
324 TCGv_i32 t0
= tcg_temp_new_i32();
325 tcg_gen_op3_i32(INDEX_op_divu_i32
, t0
, arg1
, arg2
);
326 tcg_gen_mul_i32(t0
, t0
, arg2
);
327 tcg_gen_sub_i32(ret
, arg1
, t0
);
328 tcg_temp_free_i32(t0
);
329 } else if (TCG_TARGET_HAS_div2_i32
) {
330 TCGv_i32 t0
= tcg_temp_new_i32();
331 tcg_gen_movi_i32(t0
, 0);
332 tcg_gen_op5_i32(INDEX_op_divu2_i32
, t0
, ret
, arg1
, t0
, arg2
);
333 tcg_temp_free_i32(t0
);
335 gen_helper_remu_i32(ret
, arg1
, arg2
);
339 void tcg_gen_andc_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
341 if (TCG_TARGET_HAS_andc_i32
) {
342 tcg_gen_op3_i32(INDEX_op_andc_i32
, ret
, arg1
, arg2
);
344 TCGv_i32 t0
= tcg_temp_new_i32();
345 tcg_gen_not_i32(t0
, arg2
);
346 tcg_gen_and_i32(ret
, arg1
, t0
);
347 tcg_temp_free_i32(t0
);
351 void tcg_gen_eqv_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
353 if (TCG_TARGET_HAS_eqv_i32
) {
354 tcg_gen_op3_i32(INDEX_op_eqv_i32
, ret
, arg1
, arg2
);
356 tcg_gen_xor_i32(ret
, arg1
, arg2
);
357 tcg_gen_not_i32(ret
, ret
);
361 void tcg_gen_nand_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
363 if (TCG_TARGET_HAS_nand_i32
) {
364 tcg_gen_op3_i32(INDEX_op_nand_i32
, ret
, arg1
, arg2
);
366 tcg_gen_and_i32(ret
, arg1
, arg2
);
367 tcg_gen_not_i32(ret
, ret
);
371 void tcg_gen_nor_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
373 if (TCG_TARGET_HAS_nor_i32
) {
374 tcg_gen_op3_i32(INDEX_op_nor_i32
, ret
, arg1
, arg2
);
376 tcg_gen_or_i32(ret
, arg1
, arg2
);
377 tcg_gen_not_i32(ret
, ret
);
381 void tcg_gen_orc_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
383 if (TCG_TARGET_HAS_orc_i32
) {
384 tcg_gen_op3_i32(INDEX_op_orc_i32
, ret
, arg1
, arg2
);
386 TCGv_i32 t0
= tcg_temp_new_i32();
387 tcg_gen_not_i32(t0
, arg2
);
388 tcg_gen_or_i32(ret
, arg1
, t0
);
389 tcg_temp_free_i32(t0
);
393 void tcg_gen_clz_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
395 if (TCG_TARGET_HAS_clz_i32
) {
396 tcg_gen_op3_i32(INDEX_op_clz_i32
, ret
, arg1
, arg2
);
397 } else if (TCG_TARGET_HAS_clz_i64
) {
398 TCGv_i64 t1
= tcg_temp_new_i64();
399 TCGv_i64 t2
= tcg_temp_new_i64();
400 tcg_gen_extu_i32_i64(t1
, arg1
);
401 tcg_gen_extu_i32_i64(t2
, arg2
);
402 tcg_gen_addi_i64(t2
, t2
, 32);
403 tcg_gen_clz_i64(t1
, t1
, t2
);
404 tcg_gen_extrl_i64_i32(ret
, t1
);
405 tcg_temp_free_i64(t1
);
406 tcg_temp_free_i64(t2
);
407 tcg_gen_subi_i32(ret
, ret
, 32);
409 gen_helper_clz_i32(ret
, arg1
, arg2
);
413 void tcg_gen_clzi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, uint32_t arg2
)
415 tcg_gen_clz_i32(ret
, arg1
, tcg_constant_i32(arg2
));
418 void tcg_gen_ctz_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
420 if (TCG_TARGET_HAS_ctz_i32
) {
421 tcg_gen_op3_i32(INDEX_op_ctz_i32
, ret
, arg1
, arg2
);
422 } else if (TCG_TARGET_HAS_ctz_i64
) {
423 TCGv_i64 t1
= tcg_temp_new_i64();
424 TCGv_i64 t2
= tcg_temp_new_i64();
425 tcg_gen_extu_i32_i64(t1
, arg1
);
426 tcg_gen_extu_i32_i64(t2
, arg2
);
427 tcg_gen_ctz_i64(t1
, t1
, t2
);
428 tcg_gen_extrl_i64_i32(ret
, t1
);
429 tcg_temp_free_i64(t1
);
430 tcg_temp_free_i64(t2
);
431 } else if (TCG_TARGET_HAS_ctpop_i32
432 || TCG_TARGET_HAS_ctpop_i64
433 || TCG_TARGET_HAS_clz_i32
434 || TCG_TARGET_HAS_clz_i64
) {
435 TCGv_i32 z
, t
= tcg_temp_new_i32();
437 if (TCG_TARGET_HAS_ctpop_i32
|| TCG_TARGET_HAS_ctpop_i64
) {
438 tcg_gen_subi_i32(t
, arg1
, 1);
439 tcg_gen_andc_i32(t
, t
, arg1
);
440 tcg_gen_ctpop_i32(t
, t
);
442 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
443 tcg_gen_neg_i32(t
, arg1
);
444 tcg_gen_and_i32(t
, t
, arg1
);
445 tcg_gen_clzi_i32(t
, t
, 32);
446 tcg_gen_xori_i32(t
, t
, 31);
448 z
= tcg_constant_i32(0);
449 tcg_gen_movcond_i32(TCG_COND_EQ
, ret
, arg1
, z
, arg2
, t
);
450 tcg_temp_free_i32(t
);
452 gen_helper_ctz_i32(ret
, arg1
, arg2
);
456 void tcg_gen_ctzi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, uint32_t arg2
)
458 if (!TCG_TARGET_HAS_ctz_i32
&& TCG_TARGET_HAS_ctpop_i32
&& arg2
== 32) {
459 /* This equivalence has the advantage of not requiring a fixup. */
460 TCGv_i32 t
= tcg_temp_new_i32();
461 tcg_gen_subi_i32(t
, arg1
, 1);
462 tcg_gen_andc_i32(t
, t
, arg1
);
463 tcg_gen_ctpop_i32(ret
, t
);
464 tcg_temp_free_i32(t
);
466 tcg_gen_ctz_i32(ret
, arg1
, tcg_constant_i32(arg2
));
470 void tcg_gen_clrsb_i32(TCGv_i32 ret
, TCGv_i32 arg
)
472 if (TCG_TARGET_HAS_clz_i32
) {
473 TCGv_i32 t
= tcg_temp_new_i32();
474 tcg_gen_sari_i32(t
, arg
, 31);
475 tcg_gen_xor_i32(t
, t
, arg
);
476 tcg_gen_clzi_i32(t
, t
, 32);
477 tcg_gen_subi_i32(ret
, t
, 1);
478 tcg_temp_free_i32(t
);
480 gen_helper_clrsb_i32(ret
, arg
);
484 void tcg_gen_ctpop_i32(TCGv_i32 ret
, TCGv_i32 arg1
)
486 if (TCG_TARGET_HAS_ctpop_i32
) {
487 tcg_gen_op2_i32(INDEX_op_ctpop_i32
, ret
, arg1
);
488 } else if (TCG_TARGET_HAS_ctpop_i64
) {
489 TCGv_i64 t
= tcg_temp_new_i64();
490 tcg_gen_extu_i32_i64(t
, arg1
);
491 tcg_gen_ctpop_i64(t
, t
);
492 tcg_gen_extrl_i64_i32(ret
, t
);
493 tcg_temp_free_i64(t
);
495 gen_helper_ctpop_i32(ret
, arg1
);
499 void tcg_gen_rotl_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
501 if (TCG_TARGET_HAS_rot_i32
) {
502 tcg_gen_op3_i32(INDEX_op_rotl_i32
, ret
, arg1
, arg2
);
506 t0
= tcg_temp_new_i32();
507 t1
= tcg_temp_new_i32();
508 tcg_gen_shl_i32(t0
, arg1
, arg2
);
509 tcg_gen_subfi_i32(t1
, 32, arg2
);
510 tcg_gen_shr_i32(t1
, arg1
, t1
);
511 tcg_gen_or_i32(ret
, t0
, t1
);
512 tcg_temp_free_i32(t0
);
513 tcg_temp_free_i32(t1
);
517 void tcg_gen_rotli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
519 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
520 /* some cases can be optimized here */
522 tcg_gen_mov_i32(ret
, arg1
);
523 } else if (TCG_TARGET_HAS_rot_i32
) {
524 tcg_gen_rotl_i32(ret
, arg1
, tcg_constant_i32(arg2
));
527 t0
= tcg_temp_new_i32();
528 t1
= tcg_temp_new_i32();
529 tcg_gen_shli_i32(t0
, arg1
, arg2
);
530 tcg_gen_shri_i32(t1
, arg1
, 32 - arg2
);
531 tcg_gen_or_i32(ret
, t0
, t1
);
532 tcg_temp_free_i32(t0
);
533 tcg_temp_free_i32(t1
);
537 void tcg_gen_rotr_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
539 if (TCG_TARGET_HAS_rot_i32
) {
540 tcg_gen_op3_i32(INDEX_op_rotr_i32
, ret
, arg1
, arg2
);
544 t0
= tcg_temp_new_i32();
545 t1
= tcg_temp_new_i32();
546 tcg_gen_shr_i32(t0
, arg1
, arg2
);
547 tcg_gen_subfi_i32(t1
, 32, arg2
);
548 tcg_gen_shl_i32(t1
, arg1
, t1
);
549 tcg_gen_or_i32(ret
, t0
, t1
);
550 tcg_temp_free_i32(t0
);
551 tcg_temp_free_i32(t1
);
555 void tcg_gen_rotri_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
557 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
558 /* some cases can be optimized here */
560 tcg_gen_mov_i32(ret
, arg1
);
562 tcg_gen_rotli_i32(ret
, arg1
, 32 - arg2
);
566 void tcg_gen_deposit_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
,
567 unsigned int ofs
, unsigned int len
)
572 tcg_debug_assert(ofs
< 32);
573 tcg_debug_assert(len
> 0);
574 tcg_debug_assert(len
<= 32);
575 tcg_debug_assert(ofs
+ len
<= 32);
578 tcg_gen_mov_i32(ret
, arg2
);
581 if (TCG_TARGET_HAS_deposit_i32
&& TCG_TARGET_deposit_i32_valid(ofs
, len
)) {
582 tcg_gen_op5ii_i32(INDEX_op_deposit_i32
, ret
, arg1
, arg2
, ofs
, len
);
586 t1
= tcg_temp_new_i32();
588 if (TCG_TARGET_HAS_extract2_i32
) {
589 if (ofs
+ len
== 32) {
590 tcg_gen_shli_i32(t1
, arg1
, len
);
591 tcg_gen_extract2_i32(ret
, t1
, arg2
, len
);
595 tcg_gen_extract2_i32(ret
, arg1
, arg2
, len
);
596 tcg_gen_rotli_i32(ret
, ret
, len
);
601 mask
= (1u << len
) - 1;
602 if (ofs
+ len
< 32) {
603 tcg_gen_andi_i32(t1
, arg2
, mask
);
604 tcg_gen_shli_i32(t1
, t1
, ofs
);
606 tcg_gen_shli_i32(t1
, arg2
, ofs
);
608 tcg_gen_andi_i32(ret
, arg1
, ~(mask
<< ofs
));
609 tcg_gen_or_i32(ret
, ret
, t1
);
611 tcg_temp_free_i32(t1
);
614 void tcg_gen_deposit_z_i32(TCGv_i32 ret
, TCGv_i32 arg
,
615 unsigned int ofs
, unsigned int len
)
617 tcg_debug_assert(ofs
< 32);
618 tcg_debug_assert(len
> 0);
619 tcg_debug_assert(len
<= 32);
620 tcg_debug_assert(ofs
+ len
<= 32);
622 if (ofs
+ len
== 32) {
623 tcg_gen_shli_i32(ret
, arg
, ofs
);
624 } else if (ofs
== 0) {
625 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
626 } else if (TCG_TARGET_HAS_deposit_i32
627 && TCG_TARGET_deposit_i32_valid(ofs
, len
)) {
628 TCGv_i32 zero
= tcg_constant_i32(0);
629 tcg_gen_op5ii_i32(INDEX_op_deposit_i32
, ret
, zero
, arg
, ofs
, len
);
631 /* To help two-operand hosts we prefer to zero-extend first,
632 which allows ARG to stay live. */
635 if (TCG_TARGET_HAS_ext16u_i32
) {
636 tcg_gen_ext16u_i32(ret
, arg
);
637 tcg_gen_shli_i32(ret
, ret
, ofs
);
642 if (TCG_TARGET_HAS_ext8u_i32
) {
643 tcg_gen_ext8u_i32(ret
, arg
);
644 tcg_gen_shli_i32(ret
, ret
, ofs
);
649 /* Otherwise prefer zero-extension over AND for code size. */
652 if (TCG_TARGET_HAS_ext16u_i32
) {
653 tcg_gen_shli_i32(ret
, arg
, ofs
);
654 tcg_gen_ext16u_i32(ret
, ret
);
659 if (TCG_TARGET_HAS_ext8u_i32
) {
660 tcg_gen_shli_i32(ret
, arg
, ofs
);
661 tcg_gen_ext8u_i32(ret
, ret
);
666 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
667 tcg_gen_shli_i32(ret
, ret
, ofs
);
671 void tcg_gen_extract_i32(TCGv_i32 ret
, TCGv_i32 arg
,
672 unsigned int ofs
, unsigned int len
)
674 tcg_debug_assert(ofs
< 32);
675 tcg_debug_assert(len
> 0);
676 tcg_debug_assert(len
<= 32);
677 tcg_debug_assert(ofs
+ len
<= 32);
679 /* Canonicalize certain special cases, even if extract is supported. */
680 if (ofs
+ len
== 32) {
681 tcg_gen_shri_i32(ret
, arg
, 32 - len
);
685 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
689 if (TCG_TARGET_HAS_extract_i32
690 && TCG_TARGET_extract_i32_valid(ofs
, len
)) {
691 tcg_gen_op4ii_i32(INDEX_op_extract_i32
, ret
, arg
, ofs
, len
);
695 /* Assume that zero-extension, if available, is cheaper than a shift. */
698 if (TCG_TARGET_HAS_ext16u_i32
) {
699 tcg_gen_ext16u_i32(ret
, arg
);
700 tcg_gen_shri_i32(ret
, ret
, ofs
);
705 if (TCG_TARGET_HAS_ext8u_i32
) {
706 tcg_gen_ext8u_i32(ret
, arg
);
707 tcg_gen_shri_i32(ret
, ret
, ofs
);
713 /* ??? Ideally we'd know what values are available for immediate AND.
714 Assume that 8 bits are available, plus the special case of 16,
715 so that we get ext8u, ext16u. */
717 case 1 ... 8: case 16:
718 tcg_gen_shri_i32(ret
, arg
, ofs
);
719 tcg_gen_andi_i32(ret
, ret
, (1u << len
) - 1);
722 tcg_gen_shli_i32(ret
, arg
, 32 - len
- ofs
);
723 tcg_gen_shri_i32(ret
, ret
, 32 - len
);
728 void tcg_gen_sextract_i32(TCGv_i32 ret
, TCGv_i32 arg
,
729 unsigned int ofs
, unsigned int len
)
731 tcg_debug_assert(ofs
< 32);
732 tcg_debug_assert(len
> 0);
733 tcg_debug_assert(len
<= 32);
734 tcg_debug_assert(ofs
+ len
<= 32);
736 /* Canonicalize certain special cases, even if extract is supported. */
737 if (ofs
+ len
== 32) {
738 tcg_gen_sari_i32(ret
, arg
, 32 - len
);
744 tcg_gen_ext16s_i32(ret
, arg
);
747 tcg_gen_ext8s_i32(ret
, arg
);
752 if (TCG_TARGET_HAS_sextract_i32
753 && TCG_TARGET_extract_i32_valid(ofs
, len
)) {
754 tcg_gen_op4ii_i32(INDEX_op_sextract_i32
, ret
, arg
, ofs
, len
);
758 /* Assume that sign-extension, if available, is cheaper than a shift. */
761 if (TCG_TARGET_HAS_ext16s_i32
) {
762 tcg_gen_ext16s_i32(ret
, arg
);
763 tcg_gen_sari_i32(ret
, ret
, ofs
);
768 if (TCG_TARGET_HAS_ext8s_i32
) {
769 tcg_gen_ext8s_i32(ret
, arg
);
770 tcg_gen_sari_i32(ret
, ret
, ofs
);
777 if (TCG_TARGET_HAS_ext16s_i32
) {
778 tcg_gen_shri_i32(ret
, arg
, ofs
);
779 tcg_gen_ext16s_i32(ret
, ret
);
784 if (TCG_TARGET_HAS_ext8s_i32
) {
785 tcg_gen_shri_i32(ret
, arg
, ofs
);
786 tcg_gen_ext8s_i32(ret
, ret
);
792 tcg_gen_shli_i32(ret
, arg
, 32 - len
- ofs
);
793 tcg_gen_sari_i32(ret
, ret
, 32 - len
);
797 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
798 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
800 void tcg_gen_extract2_i32(TCGv_i32 ret
, TCGv_i32 al
, TCGv_i32 ah
,
803 tcg_debug_assert(ofs
<= 32);
805 tcg_gen_mov_i32(ret
, al
);
806 } else if (ofs
== 32) {
807 tcg_gen_mov_i32(ret
, ah
);
808 } else if (al
== ah
) {
809 tcg_gen_rotri_i32(ret
, al
, ofs
);
810 } else if (TCG_TARGET_HAS_extract2_i32
) {
811 tcg_gen_op4i_i32(INDEX_op_extract2_i32
, ret
, al
, ah
, ofs
);
813 TCGv_i32 t0
= tcg_temp_new_i32();
814 tcg_gen_shri_i32(t0
, al
, ofs
);
815 tcg_gen_deposit_i32(ret
, t0
, ah
, 32 - ofs
, ofs
);
816 tcg_temp_free_i32(t0
);
820 void tcg_gen_movcond_i32(TCGCond cond
, TCGv_i32 ret
, TCGv_i32 c1
,
821 TCGv_i32 c2
, TCGv_i32 v1
, TCGv_i32 v2
)
823 if (cond
== TCG_COND_ALWAYS
) {
824 tcg_gen_mov_i32(ret
, v1
);
825 } else if (cond
== TCG_COND_NEVER
) {
826 tcg_gen_mov_i32(ret
, v2
);
827 } else if (TCG_TARGET_HAS_movcond_i32
) {
828 tcg_gen_op6i_i32(INDEX_op_movcond_i32
, ret
, c1
, c2
, v1
, v2
, cond
);
830 TCGv_i32 t0
= tcg_temp_new_i32();
831 TCGv_i32 t1
= tcg_temp_new_i32();
832 tcg_gen_setcond_i32(cond
, t0
, c1
, c2
);
833 tcg_gen_neg_i32(t0
, t0
);
834 tcg_gen_and_i32(t1
, v1
, t0
);
835 tcg_gen_andc_i32(ret
, v2
, t0
);
836 tcg_gen_or_i32(ret
, ret
, t1
);
837 tcg_temp_free_i32(t0
);
838 tcg_temp_free_i32(t1
);
842 void tcg_gen_add2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 al
,
843 TCGv_i32 ah
, TCGv_i32 bl
, TCGv_i32 bh
)
845 if (TCG_TARGET_HAS_add2_i32
) {
846 tcg_gen_op6_i32(INDEX_op_add2_i32
, rl
, rh
, al
, ah
, bl
, bh
);
848 TCGv_i64 t0
= tcg_temp_new_i64();
849 TCGv_i64 t1
= tcg_temp_new_i64();
850 tcg_gen_concat_i32_i64(t0
, al
, ah
);
851 tcg_gen_concat_i32_i64(t1
, bl
, bh
);
852 tcg_gen_add_i64(t0
, t0
, t1
);
853 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
854 tcg_temp_free_i64(t0
);
855 tcg_temp_free_i64(t1
);
859 void tcg_gen_sub2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 al
,
860 TCGv_i32 ah
, TCGv_i32 bl
, TCGv_i32 bh
)
862 if (TCG_TARGET_HAS_sub2_i32
) {
863 tcg_gen_op6_i32(INDEX_op_sub2_i32
, rl
, rh
, al
, ah
, bl
, bh
);
865 TCGv_i64 t0
= tcg_temp_new_i64();
866 TCGv_i64 t1
= tcg_temp_new_i64();
867 tcg_gen_concat_i32_i64(t0
, al
, ah
);
868 tcg_gen_concat_i32_i64(t1
, bl
, bh
);
869 tcg_gen_sub_i64(t0
, t0
, t1
);
870 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
871 tcg_temp_free_i64(t0
);
872 tcg_temp_free_i64(t1
);
876 void tcg_gen_mulu2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
878 if (TCG_TARGET_HAS_mulu2_i32
) {
879 tcg_gen_op4_i32(INDEX_op_mulu2_i32
, rl
, rh
, arg1
, arg2
);
880 } else if (TCG_TARGET_HAS_muluh_i32
) {
881 TCGv_i32 t
= tcg_temp_new_i32();
882 tcg_gen_op3_i32(INDEX_op_mul_i32
, t
, arg1
, arg2
);
883 tcg_gen_op3_i32(INDEX_op_muluh_i32
, rh
, arg1
, arg2
);
884 tcg_gen_mov_i32(rl
, t
);
885 tcg_temp_free_i32(t
);
887 TCGv_i64 t0
= tcg_temp_new_i64();
888 TCGv_i64 t1
= tcg_temp_new_i64();
889 tcg_gen_extu_i32_i64(t0
, arg1
);
890 tcg_gen_extu_i32_i64(t1
, arg2
);
891 tcg_gen_mul_i64(t0
, t0
, t1
);
892 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
893 tcg_temp_free_i64(t0
);
894 tcg_temp_free_i64(t1
);
898 void tcg_gen_muls2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
900 if (TCG_TARGET_HAS_muls2_i32
) {
901 tcg_gen_op4_i32(INDEX_op_muls2_i32
, rl
, rh
, arg1
, arg2
);
902 } else if (TCG_TARGET_HAS_mulsh_i32
) {
903 TCGv_i32 t
= tcg_temp_new_i32();
904 tcg_gen_op3_i32(INDEX_op_mul_i32
, t
, arg1
, arg2
);
905 tcg_gen_op3_i32(INDEX_op_mulsh_i32
, rh
, arg1
, arg2
);
906 tcg_gen_mov_i32(rl
, t
);
907 tcg_temp_free_i32(t
);
908 } else if (TCG_TARGET_REG_BITS
== 32) {
909 TCGv_i32 t0
= tcg_temp_new_i32();
910 TCGv_i32 t1
= tcg_temp_new_i32();
911 TCGv_i32 t2
= tcg_temp_new_i32();
912 TCGv_i32 t3
= tcg_temp_new_i32();
913 tcg_gen_mulu2_i32(t0
, t1
, arg1
, arg2
);
914 /* Adjust for negative inputs. */
915 tcg_gen_sari_i32(t2
, arg1
, 31);
916 tcg_gen_sari_i32(t3
, arg2
, 31);
917 tcg_gen_and_i32(t2
, t2
, arg2
);
918 tcg_gen_and_i32(t3
, t3
, arg1
);
919 tcg_gen_sub_i32(rh
, t1
, t2
);
920 tcg_gen_sub_i32(rh
, rh
, t3
);
921 tcg_gen_mov_i32(rl
, t0
);
922 tcg_temp_free_i32(t0
);
923 tcg_temp_free_i32(t1
);
924 tcg_temp_free_i32(t2
);
925 tcg_temp_free_i32(t3
);
927 TCGv_i64 t0
= tcg_temp_new_i64();
928 TCGv_i64 t1
= tcg_temp_new_i64();
929 tcg_gen_ext_i32_i64(t0
, arg1
);
930 tcg_gen_ext_i32_i64(t1
, arg2
);
931 tcg_gen_mul_i64(t0
, t0
, t1
);
932 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
933 tcg_temp_free_i64(t0
);
934 tcg_temp_free_i64(t1
);
938 void tcg_gen_mulsu2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
940 if (TCG_TARGET_REG_BITS
== 32) {
941 TCGv_i32 t0
= tcg_temp_new_i32();
942 TCGv_i32 t1
= tcg_temp_new_i32();
943 TCGv_i32 t2
= tcg_temp_new_i32();
944 tcg_gen_mulu2_i32(t0
, t1
, arg1
, arg2
);
945 /* Adjust for negative input for the signed arg1. */
946 tcg_gen_sari_i32(t2
, arg1
, 31);
947 tcg_gen_and_i32(t2
, t2
, arg2
);
948 tcg_gen_sub_i32(rh
, t1
, t2
);
949 tcg_gen_mov_i32(rl
, t0
);
950 tcg_temp_free_i32(t0
);
951 tcg_temp_free_i32(t1
);
952 tcg_temp_free_i32(t2
);
954 TCGv_i64 t0
= tcg_temp_new_i64();
955 TCGv_i64 t1
= tcg_temp_new_i64();
956 tcg_gen_ext_i32_i64(t0
, arg1
);
957 tcg_gen_extu_i32_i64(t1
, arg2
);
958 tcg_gen_mul_i64(t0
, t0
, t1
);
959 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
960 tcg_temp_free_i64(t0
);
961 tcg_temp_free_i64(t1
);
965 void tcg_gen_ext8s_i32(TCGv_i32 ret
, TCGv_i32 arg
)
967 if (TCG_TARGET_HAS_ext8s_i32
) {
968 tcg_gen_op2_i32(INDEX_op_ext8s_i32
, ret
, arg
);
970 tcg_gen_shli_i32(ret
, arg
, 24);
971 tcg_gen_sari_i32(ret
, ret
, 24);
975 void tcg_gen_ext16s_i32(TCGv_i32 ret
, TCGv_i32 arg
)
977 if (TCG_TARGET_HAS_ext16s_i32
) {
978 tcg_gen_op2_i32(INDEX_op_ext16s_i32
, ret
, arg
);
980 tcg_gen_shli_i32(ret
, arg
, 16);
981 tcg_gen_sari_i32(ret
, ret
, 16);
985 void tcg_gen_ext8u_i32(TCGv_i32 ret
, TCGv_i32 arg
)
987 if (TCG_TARGET_HAS_ext8u_i32
) {
988 tcg_gen_op2_i32(INDEX_op_ext8u_i32
, ret
, arg
);
990 tcg_gen_andi_i32(ret
, arg
, 0xffu
);
994 void tcg_gen_ext16u_i32(TCGv_i32 ret
, TCGv_i32 arg
)
996 if (TCG_TARGET_HAS_ext16u_i32
) {
997 tcg_gen_op2_i32(INDEX_op_ext16u_i32
, ret
, arg
);
999 tcg_gen_andi_i32(ret
, arg
, 0xffffu
);
1003 void tcg_gen_bswap16_i32(TCGv_i32 ret
, TCGv_i32 arg
, int flags
)
1005 /* Only one extension flag may be present. */
1006 tcg_debug_assert(!(flags
& TCG_BSWAP_OS
) || !(flags
& TCG_BSWAP_OZ
));
1008 if (TCG_TARGET_HAS_bswap16_i32
) {
1009 tcg_gen_op3i_i32(INDEX_op_bswap16_i32
, ret
, arg
, flags
);
1011 TCGv_i32 t0
= tcg_temp_new_i32();
1012 TCGv_i32 t1
= tcg_temp_new_i32();
1014 tcg_gen_shri_i32(t0
, arg
, 8);
1015 if (!(flags
& TCG_BSWAP_IZ
)) {
1016 tcg_gen_ext8u_i32(t0
, t0
);
1019 if (flags
& TCG_BSWAP_OS
) {
1020 tcg_gen_shli_i32(t1
, arg
, 24);
1021 tcg_gen_sari_i32(t1
, t1
, 16);
1022 } else if (flags
& TCG_BSWAP_OZ
) {
1023 tcg_gen_ext8u_i32(t1
, arg
);
1024 tcg_gen_shli_i32(t1
, t1
, 8);
1026 tcg_gen_shli_i32(t1
, arg
, 8);
1029 tcg_gen_or_i32(ret
, t0
, t1
);
1030 tcg_temp_free_i32(t0
);
1031 tcg_temp_free_i32(t1
);
1035 void tcg_gen_bswap32_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1037 if (TCG_TARGET_HAS_bswap32_i32
) {
1038 tcg_gen_op3i_i32(INDEX_op_bswap32_i32
, ret
, arg
, 0);
1040 TCGv_i32 t0
= tcg_temp_new_i32();
1041 TCGv_i32 t1
= tcg_temp_new_i32();
1042 TCGv_i32 t2
= tcg_constant_i32(0x00ff00ff);
1045 tcg_gen_shri_i32(t0
, arg
, 8); /* t0 = .abc */
1046 tcg_gen_and_i32(t1
, arg
, t2
); /* t1 = .b.d */
1047 tcg_gen_and_i32(t0
, t0
, t2
); /* t0 = .a.c */
1048 tcg_gen_shli_i32(t1
, t1
, 8); /* t1 = b.d. */
1049 tcg_gen_or_i32(ret
, t0
, t1
); /* ret = badc */
1051 tcg_gen_shri_i32(t0
, ret
, 16); /* t0 = ..ba */
1052 tcg_gen_shli_i32(t1
, ret
, 16); /* t1 = dc.. */
1053 tcg_gen_or_i32(ret
, t0
, t1
); /* ret = dcba */
1055 tcg_temp_free_i32(t0
);
1056 tcg_temp_free_i32(t1
);
1060 void tcg_gen_smin_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1062 tcg_gen_movcond_i32(TCG_COND_LT
, ret
, a
, b
, a
, b
);
1065 void tcg_gen_umin_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1067 tcg_gen_movcond_i32(TCG_COND_LTU
, ret
, a
, b
, a
, b
);
1070 void tcg_gen_smax_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1072 tcg_gen_movcond_i32(TCG_COND_LT
, ret
, a
, b
, b
, a
);
1075 void tcg_gen_umax_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1077 tcg_gen_movcond_i32(TCG_COND_LTU
, ret
, a
, b
, b
, a
);
1080 void tcg_gen_abs_i32(TCGv_i32 ret
, TCGv_i32 a
)
1082 TCGv_i32 t
= tcg_temp_new_i32();
1084 tcg_gen_sari_i32(t
, a
, 31);
1085 tcg_gen_xor_i32(ret
, a
, t
);
1086 tcg_gen_sub_i32(ret
, ret
, t
);
1087 tcg_temp_free_i32(t
);
1092 #if TCG_TARGET_REG_BITS == 32
1093 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1095 void tcg_gen_discard_i64(TCGv_i64 arg
)
1097 tcg_gen_discard_i32(TCGV_LOW(arg
));
1098 tcg_gen_discard_i32(TCGV_HIGH(arg
));
1101 void tcg_gen_mov_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1103 TCGTemp
*ts
= tcgv_i64_temp(arg
);
1105 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1106 if (ts
->kind
== TEMP_CONST
) {
1107 tcg_gen_movi_i64(ret
, ts
->val
);
1109 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1110 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
));
1114 void tcg_gen_movi_i64(TCGv_i64 ret
, int64_t arg
)
1116 tcg_gen_movi_i32(TCGV_LOW(ret
), arg
);
1117 tcg_gen_movi_i32(TCGV_HIGH(ret
), arg
>> 32);
1120 void tcg_gen_ld8u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1122 tcg_gen_ld8u_i32(TCGV_LOW(ret
), arg2
, offset
);
1123 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1126 void tcg_gen_ld8s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1128 tcg_gen_ld8s_i32(TCGV_LOW(ret
), arg2
, offset
);
1129 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1132 void tcg_gen_ld16u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1134 tcg_gen_ld16u_i32(TCGV_LOW(ret
), arg2
, offset
);
1135 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1138 void tcg_gen_ld16s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1140 tcg_gen_ld16s_i32(TCGV_LOW(ret
), arg2
, offset
);
1141 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1144 void tcg_gen_ld32u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1146 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1147 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1150 void tcg_gen_ld32s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1152 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1153 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1156 void tcg_gen_ld_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1158 /* Since arg2 and ret have different types,
1159 they cannot be the same temporary */
1160 #ifdef HOST_WORDS_BIGENDIAN
1161 tcg_gen_ld_i32(TCGV_HIGH(ret
), arg2
, offset
);
1162 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
+ 4);
1164 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1165 tcg_gen_ld_i32(TCGV_HIGH(ret
), arg2
, offset
+ 4);
1169 void tcg_gen_st_i64(TCGv_i64 arg1
, TCGv_ptr arg2
, tcg_target_long offset
)
1171 #ifdef HOST_WORDS_BIGENDIAN
1172 tcg_gen_st_i32(TCGV_HIGH(arg1
), arg2
, offset
);
1173 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
+ 4);
1175 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
);
1176 tcg_gen_st_i32(TCGV_HIGH(arg1
), arg2
, offset
+ 4);
1180 void tcg_gen_and_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1182 tcg_gen_and_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1183 tcg_gen_and_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1186 void tcg_gen_or_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1188 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1189 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1192 void tcg_gen_xor_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1194 tcg_gen_xor_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1195 tcg_gen_xor_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1198 void tcg_gen_shl_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1200 gen_helper_shl_i64(ret
, arg1
, arg2
);
1203 void tcg_gen_shr_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1205 gen_helper_shr_i64(ret
, arg1
, arg2
);
1208 void tcg_gen_sar_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1210 gen_helper_sar_i64(ret
, arg1
, arg2
);
1213 void tcg_gen_mul_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1218 t0
= tcg_temp_new_i64();
1219 t1
= tcg_temp_new_i32();
1221 tcg_gen_mulu2_i32(TCGV_LOW(t0
), TCGV_HIGH(t0
),
1222 TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1224 tcg_gen_mul_i32(t1
, TCGV_LOW(arg1
), TCGV_HIGH(arg2
));
1225 tcg_gen_add_i32(TCGV_HIGH(t0
), TCGV_HIGH(t0
), t1
);
1226 tcg_gen_mul_i32(t1
, TCGV_HIGH(arg1
), TCGV_LOW(arg2
));
1227 tcg_gen_add_i32(TCGV_HIGH(t0
), TCGV_HIGH(t0
), t1
);
1229 tcg_gen_mov_i64(ret
, t0
);
1230 tcg_temp_free_i64(t0
);
1231 tcg_temp_free_i32(t1
);
1236 void tcg_gen_movi_i64(TCGv_i64 ret
, int64_t arg
)
1238 tcg_gen_mov_i64(ret
, tcg_constant_i64(arg
));
1241 #endif /* TCG_TARGET_REG_SIZE == 32 */
1243 void tcg_gen_addi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1245 /* some cases can be optimized here */
1247 tcg_gen_mov_i64(ret
, arg1
);
1248 } else if (TCG_TARGET_REG_BITS
== 64) {
1249 tcg_gen_add_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1251 tcg_gen_add2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1252 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1253 tcg_constant_i32(arg2
), tcg_constant_i32(arg2
>> 32));
1257 void tcg_gen_subfi_i64(TCGv_i64 ret
, int64_t arg1
, TCGv_i64 arg2
)
1259 if (arg1
== 0 && TCG_TARGET_HAS_neg_i64
) {
1260 /* Don't recurse with tcg_gen_neg_i64. */
1261 tcg_gen_op2_i64(INDEX_op_neg_i64
, ret
, arg2
);
1262 } else if (TCG_TARGET_REG_BITS
== 64) {
1263 tcg_gen_sub_i64(ret
, tcg_constant_i64(arg1
), arg2
);
1265 tcg_gen_sub2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1266 tcg_constant_i32(arg1
), tcg_constant_i32(arg1
>> 32),
1267 TCGV_LOW(arg2
), TCGV_HIGH(arg2
));
1271 void tcg_gen_subi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1273 /* some cases can be optimized here */
1275 tcg_gen_mov_i64(ret
, arg1
);
1276 } else if (TCG_TARGET_REG_BITS
== 64) {
1277 tcg_gen_sub_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1279 tcg_gen_sub2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1280 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1281 tcg_constant_i32(arg2
), tcg_constant_i32(arg2
>> 32));
1285 void tcg_gen_andi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1287 if (TCG_TARGET_REG_BITS
== 32) {
1288 tcg_gen_andi_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1289 tcg_gen_andi_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1293 /* Some cases can be optimized here. */
1296 tcg_gen_movi_i64(ret
, 0);
1299 tcg_gen_mov_i64(ret
, arg1
);
1302 /* Don't recurse with tcg_gen_ext8u_i64. */
1303 if (TCG_TARGET_HAS_ext8u_i64
) {
1304 tcg_gen_op2_i64(INDEX_op_ext8u_i64
, ret
, arg1
);
1309 if (TCG_TARGET_HAS_ext16u_i64
) {
1310 tcg_gen_op2_i64(INDEX_op_ext16u_i64
, ret
, arg1
);
1315 if (TCG_TARGET_HAS_ext32u_i64
) {
1316 tcg_gen_op2_i64(INDEX_op_ext32u_i64
, ret
, arg1
);
1322 tcg_gen_and_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1325 void tcg_gen_ori_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1327 if (TCG_TARGET_REG_BITS
== 32) {
1328 tcg_gen_ori_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1329 tcg_gen_ori_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1332 /* Some cases can be optimized here. */
1334 tcg_gen_movi_i64(ret
, -1);
1335 } else if (arg2
== 0) {
1336 tcg_gen_mov_i64(ret
, arg1
);
1338 tcg_gen_or_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1342 void tcg_gen_xori_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1344 if (TCG_TARGET_REG_BITS
== 32) {
1345 tcg_gen_xori_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1346 tcg_gen_xori_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1349 /* Some cases can be optimized here. */
1351 tcg_gen_mov_i64(ret
, arg1
);
1352 } else if (arg2
== -1 && TCG_TARGET_HAS_not_i64
) {
1353 /* Don't recurse with tcg_gen_not_i64. */
1354 tcg_gen_op2_i64(INDEX_op_not_i64
, ret
, arg1
);
1356 tcg_gen_xor_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1360 static inline void tcg_gen_shifti_i64(TCGv_i64 ret
, TCGv_i64 arg1
,
1361 unsigned c
, bool right
, bool arith
)
1363 tcg_debug_assert(c
< 64);
1365 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
1366 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
1367 } else if (c
>= 32) {
1371 tcg_gen_sari_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), c
);
1372 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), 31);
1374 tcg_gen_shri_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), c
);
1375 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1378 tcg_gen_shli_i32(TCGV_HIGH(ret
), TCGV_LOW(arg1
), c
);
1379 tcg_gen_movi_i32(TCGV_LOW(ret
), 0);
1382 if (TCG_TARGET_HAS_extract2_i32
) {
1383 tcg_gen_extract2_i32(TCGV_LOW(ret
),
1384 TCGV_LOW(arg1
), TCGV_HIGH(arg1
), c
);
1386 tcg_gen_shri_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), c
);
1387 tcg_gen_deposit_i32(TCGV_LOW(ret
), TCGV_LOW(ret
),
1388 TCGV_HIGH(arg1
), 32 - c
, c
);
1391 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), c
);
1393 tcg_gen_shri_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), c
);
1396 if (TCG_TARGET_HAS_extract2_i32
) {
1397 tcg_gen_extract2_i32(TCGV_HIGH(ret
),
1398 TCGV_LOW(arg1
), TCGV_HIGH(arg1
), 32 - c
);
1400 TCGv_i32 t0
= tcg_temp_new_i32();
1401 tcg_gen_shri_i32(t0
, TCGV_LOW(arg1
), 32 - c
);
1402 tcg_gen_deposit_i32(TCGV_HIGH(ret
), t0
,
1403 TCGV_HIGH(arg1
), c
, 32 - c
);
1404 tcg_temp_free_i32(t0
);
1406 tcg_gen_shli_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), c
);
1410 void tcg_gen_shli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1412 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1413 if (TCG_TARGET_REG_BITS
== 32) {
1414 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 0, 0);
1415 } else if (arg2
== 0) {
1416 tcg_gen_mov_i64(ret
, arg1
);
1418 tcg_gen_shl_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1422 void tcg_gen_shri_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1424 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1425 if (TCG_TARGET_REG_BITS
== 32) {
1426 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 1, 0);
1427 } else if (arg2
== 0) {
1428 tcg_gen_mov_i64(ret
, arg1
);
1430 tcg_gen_shr_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1434 void tcg_gen_sari_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1436 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1437 if (TCG_TARGET_REG_BITS
== 32) {
1438 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 1, 1);
1439 } else if (arg2
== 0) {
1440 tcg_gen_mov_i64(ret
, arg1
);
1442 tcg_gen_sar_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1446 void tcg_gen_brcond_i64(TCGCond cond
, TCGv_i64 arg1
, TCGv_i64 arg2
, TCGLabel
*l
)
1448 if (cond
== TCG_COND_ALWAYS
) {
1450 } else if (cond
!= TCG_COND_NEVER
) {
1452 if (TCG_TARGET_REG_BITS
== 32) {
1453 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32
, TCGV_LOW(arg1
),
1454 TCGV_HIGH(arg1
), TCGV_LOW(arg2
),
1455 TCGV_HIGH(arg2
), cond
, label_arg(l
));
1457 tcg_gen_op4ii_i64(INDEX_op_brcond_i64
, arg1
, arg2
, cond
,
1463 void tcg_gen_brcondi_i64(TCGCond cond
, TCGv_i64 arg1
, int64_t arg2
, TCGLabel
*l
)
1465 if (TCG_TARGET_REG_BITS
== 64) {
1466 tcg_gen_brcond_i64(cond
, arg1
, tcg_constant_i64(arg2
), l
);
1467 } else if (cond
== TCG_COND_ALWAYS
) {
1469 } else if (cond
!= TCG_COND_NEVER
) {
1471 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32
,
1472 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1473 tcg_constant_i32(arg2
),
1474 tcg_constant_i32(arg2
>> 32),
1475 cond
, label_arg(l
));
1479 void tcg_gen_setcond_i64(TCGCond cond
, TCGv_i64 ret
,
1480 TCGv_i64 arg1
, TCGv_i64 arg2
)
1482 if (cond
== TCG_COND_ALWAYS
) {
1483 tcg_gen_movi_i64(ret
, 1);
1484 } else if (cond
== TCG_COND_NEVER
) {
1485 tcg_gen_movi_i64(ret
, 0);
1487 if (TCG_TARGET_REG_BITS
== 32) {
1488 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, TCGV_LOW(ret
),
1489 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1490 TCGV_LOW(arg2
), TCGV_HIGH(arg2
), cond
);
1491 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1493 tcg_gen_op4i_i64(INDEX_op_setcond_i64
, ret
, arg1
, arg2
, cond
);
1498 void tcg_gen_setcondi_i64(TCGCond cond
, TCGv_i64 ret
,
1499 TCGv_i64 arg1
, int64_t arg2
)
1501 if (TCG_TARGET_REG_BITS
== 64) {
1502 tcg_gen_setcond_i64(cond
, ret
, arg1
, tcg_constant_i64(arg2
));
1503 } else if (cond
== TCG_COND_ALWAYS
) {
1504 tcg_gen_movi_i64(ret
, 1);
1505 } else if (cond
== TCG_COND_NEVER
) {
1506 tcg_gen_movi_i64(ret
, 0);
1508 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, TCGV_LOW(ret
),
1509 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1510 tcg_constant_i32(arg2
),
1511 tcg_constant_i32(arg2
>> 32), cond
);
1512 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1516 void tcg_gen_muli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1519 tcg_gen_movi_i64(ret
, 0);
1520 } else if (is_power_of_2(arg2
)) {
1521 tcg_gen_shli_i64(ret
, arg1
, ctz64(arg2
));
1523 TCGv_i64 t0
= tcg_const_i64(arg2
);
1524 tcg_gen_mul_i64(ret
, arg1
, t0
);
1525 tcg_temp_free_i64(t0
);
1529 void tcg_gen_div_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1531 if (TCG_TARGET_HAS_div_i64
) {
1532 tcg_gen_op3_i64(INDEX_op_div_i64
, ret
, arg1
, arg2
);
1533 } else if (TCG_TARGET_HAS_div2_i64
) {
1534 TCGv_i64 t0
= tcg_temp_new_i64();
1535 tcg_gen_sari_i64(t0
, arg1
, 63);
1536 tcg_gen_op5_i64(INDEX_op_div2_i64
, ret
, t0
, arg1
, t0
, arg2
);
1537 tcg_temp_free_i64(t0
);
1539 gen_helper_div_i64(ret
, arg1
, arg2
);
1543 void tcg_gen_rem_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1545 if (TCG_TARGET_HAS_rem_i64
) {
1546 tcg_gen_op3_i64(INDEX_op_rem_i64
, ret
, arg1
, arg2
);
1547 } else if (TCG_TARGET_HAS_div_i64
) {
1548 TCGv_i64 t0
= tcg_temp_new_i64();
1549 tcg_gen_op3_i64(INDEX_op_div_i64
, t0
, arg1
, arg2
);
1550 tcg_gen_mul_i64(t0
, t0
, arg2
);
1551 tcg_gen_sub_i64(ret
, arg1
, t0
);
1552 tcg_temp_free_i64(t0
);
1553 } else if (TCG_TARGET_HAS_div2_i64
) {
1554 TCGv_i64 t0
= tcg_temp_new_i64();
1555 tcg_gen_sari_i64(t0
, arg1
, 63);
1556 tcg_gen_op5_i64(INDEX_op_div2_i64
, t0
, ret
, arg1
, t0
, arg2
);
1557 tcg_temp_free_i64(t0
);
1559 gen_helper_rem_i64(ret
, arg1
, arg2
);
1563 void tcg_gen_divu_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1565 if (TCG_TARGET_HAS_div_i64
) {
1566 tcg_gen_op3_i64(INDEX_op_divu_i64
, ret
, arg1
, arg2
);
1567 } else if (TCG_TARGET_HAS_div2_i64
) {
1568 TCGv_i64 t0
= tcg_temp_new_i64();
1569 tcg_gen_movi_i64(t0
, 0);
1570 tcg_gen_op5_i64(INDEX_op_divu2_i64
, ret
, t0
, arg1
, t0
, arg2
);
1571 tcg_temp_free_i64(t0
);
1573 gen_helper_divu_i64(ret
, arg1
, arg2
);
1577 void tcg_gen_remu_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1579 if (TCG_TARGET_HAS_rem_i64
) {
1580 tcg_gen_op3_i64(INDEX_op_remu_i64
, ret
, arg1
, arg2
);
1581 } else if (TCG_TARGET_HAS_div_i64
) {
1582 TCGv_i64 t0
= tcg_temp_new_i64();
1583 tcg_gen_op3_i64(INDEX_op_divu_i64
, t0
, arg1
, arg2
);
1584 tcg_gen_mul_i64(t0
, t0
, arg2
);
1585 tcg_gen_sub_i64(ret
, arg1
, t0
);
1586 tcg_temp_free_i64(t0
);
1587 } else if (TCG_TARGET_HAS_div2_i64
) {
1588 TCGv_i64 t0
= tcg_temp_new_i64();
1589 tcg_gen_movi_i64(t0
, 0);
1590 tcg_gen_op5_i64(INDEX_op_divu2_i64
, t0
, ret
, arg1
, t0
, arg2
);
1591 tcg_temp_free_i64(t0
);
1593 gen_helper_remu_i64(ret
, arg1
, arg2
);
1597 void tcg_gen_ext8s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1599 if (TCG_TARGET_REG_BITS
== 32) {
1600 tcg_gen_ext8s_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1601 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1602 } else if (TCG_TARGET_HAS_ext8s_i64
) {
1603 tcg_gen_op2_i64(INDEX_op_ext8s_i64
, ret
, arg
);
1605 tcg_gen_shli_i64(ret
, arg
, 56);
1606 tcg_gen_sari_i64(ret
, ret
, 56);
1610 void tcg_gen_ext16s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1612 if (TCG_TARGET_REG_BITS
== 32) {
1613 tcg_gen_ext16s_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1614 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1615 } else if (TCG_TARGET_HAS_ext16s_i64
) {
1616 tcg_gen_op2_i64(INDEX_op_ext16s_i64
, ret
, arg
);
1618 tcg_gen_shli_i64(ret
, arg
, 48);
1619 tcg_gen_sari_i64(ret
, ret
, 48);
1623 void tcg_gen_ext32s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1625 if (TCG_TARGET_REG_BITS
== 32) {
1626 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1627 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1628 } else if (TCG_TARGET_HAS_ext32s_i64
) {
1629 tcg_gen_op2_i64(INDEX_op_ext32s_i64
, ret
, arg
);
1631 tcg_gen_shli_i64(ret
, arg
, 32);
1632 tcg_gen_sari_i64(ret
, ret
, 32);
1636 void tcg_gen_ext8u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1638 if (TCG_TARGET_REG_BITS
== 32) {
1639 tcg_gen_ext8u_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1640 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1641 } else if (TCG_TARGET_HAS_ext8u_i64
) {
1642 tcg_gen_op2_i64(INDEX_op_ext8u_i64
, ret
, arg
);
1644 tcg_gen_andi_i64(ret
, arg
, 0xffu
);
1648 void tcg_gen_ext16u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1650 if (TCG_TARGET_REG_BITS
== 32) {
1651 tcg_gen_ext16u_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1652 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1653 } else if (TCG_TARGET_HAS_ext16u_i64
) {
1654 tcg_gen_op2_i64(INDEX_op_ext16u_i64
, ret
, arg
);
1656 tcg_gen_andi_i64(ret
, arg
, 0xffffu
);
1660 void tcg_gen_ext32u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1662 if (TCG_TARGET_REG_BITS
== 32) {
1663 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1664 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1665 } else if (TCG_TARGET_HAS_ext32u_i64
) {
1666 tcg_gen_op2_i64(INDEX_op_ext32u_i64
, ret
, arg
);
1668 tcg_gen_andi_i64(ret
, arg
, 0xffffffffu
);
1672 void tcg_gen_bswap16_i64(TCGv_i64 ret
, TCGv_i64 arg
, int flags
)
1674 /* Only one extension flag may be present. */
1675 tcg_debug_assert(!(flags
& TCG_BSWAP_OS
) || !(flags
& TCG_BSWAP_OZ
));
1677 if (TCG_TARGET_REG_BITS
== 32) {
1678 tcg_gen_bswap16_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), flags
);
1679 if (flags
& TCG_BSWAP_OS
) {
1680 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1682 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1684 } else if (TCG_TARGET_HAS_bswap16_i64
) {
1685 tcg_gen_op3i_i64(INDEX_op_bswap16_i64
, ret
, arg
, flags
);
1687 TCGv_i64 t0
= tcg_temp_new_i64();
1688 TCGv_i64 t1
= tcg_temp_new_i64();
1690 tcg_gen_shri_i64(t0
, arg
, 8);
1691 if (!(flags
& TCG_BSWAP_IZ
)) {
1692 tcg_gen_ext8u_i64(t0
, t0
);
1695 if (flags
& TCG_BSWAP_OS
) {
1696 tcg_gen_shli_i64(t1
, arg
, 56);
1697 tcg_gen_sari_i64(t1
, t1
, 48);
1698 } else if (flags
& TCG_BSWAP_OZ
) {
1699 tcg_gen_ext8u_i64(t1
, arg
);
1700 tcg_gen_shli_i64(t1
, t1
, 8);
1702 tcg_gen_shli_i64(t1
, arg
, 8);
1705 tcg_gen_or_i64(ret
, t0
, t1
);
1706 tcg_temp_free_i64(t0
);
1707 tcg_temp_free_i64(t1
);
1711 void tcg_gen_bswap32_i64(TCGv_i64 ret
, TCGv_i64 arg
, int flags
)
1713 /* Only one extension flag may be present. */
1714 tcg_debug_assert(!(flags
& TCG_BSWAP_OS
) || !(flags
& TCG_BSWAP_OZ
));
1716 if (TCG_TARGET_REG_BITS
== 32) {
1717 tcg_gen_bswap32_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1718 if (flags
& TCG_BSWAP_OS
) {
1719 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1721 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1723 } else if (TCG_TARGET_HAS_bswap32_i64
) {
1724 tcg_gen_op3i_i64(INDEX_op_bswap32_i64
, ret
, arg
, flags
);
1726 TCGv_i64 t0
= tcg_temp_new_i64();
1727 TCGv_i64 t1
= tcg_temp_new_i64();
1728 TCGv_i64 t2
= tcg_constant_i64(0x00ff00ff);
1730 /* arg = xxxxabcd */
1731 tcg_gen_shri_i64(t0
, arg
, 8); /* t0 = .xxxxabc */
1732 tcg_gen_and_i64(t1
, arg
, t2
); /* t1 = .....b.d */
1733 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = .....a.c */
1734 tcg_gen_shli_i64(t1
, t1
, 8); /* t1 = ....b.d. */
1735 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = ....badc */
1737 tcg_gen_shli_i64(t1
, ret
, 48); /* t1 = dc...... */
1738 tcg_gen_shri_i64(t0
, ret
, 16); /* t0 = ......ba */
1739 if (flags
& TCG_BSWAP_OS
) {
1740 tcg_gen_sari_i64(t1
, t1
, 32); /* t1 = ssssdc.. */
1742 tcg_gen_shri_i64(t1
, t1
, 32); /* t1 = ....dc.. */
1744 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = ssssdcba */
1746 tcg_temp_free_i64(t0
);
1747 tcg_temp_free_i64(t1
);
1751 void tcg_gen_bswap64_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1753 if (TCG_TARGET_REG_BITS
== 32) {
1755 t0
= tcg_temp_new_i32();
1756 t1
= tcg_temp_new_i32();
1758 tcg_gen_bswap32_i32(t0
, TCGV_LOW(arg
));
1759 tcg_gen_bswap32_i32(t1
, TCGV_HIGH(arg
));
1760 tcg_gen_mov_i32(TCGV_LOW(ret
), t1
);
1761 tcg_gen_mov_i32(TCGV_HIGH(ret
), t0
);
1762 tcg_temp_free_i32(t0
);
1763 tcg_temp_free_i32(t1
);
1764 } else if (TCG_TARGET_HAS_bswap64_i64
) {
1765 tcg_gen_op3i_i64(INDEX_op_bswap64_i64
, ret
, arg
, 0);
1767 TCGv_i64 t0
= tcg_temp_new_i64();
1768 TCGv_i64 t1
= tcg_temp_new_i64();
1769 TCGv_i64 t2
= tcg_temp_new_i64();
1771 /* arg = abcdefgh */
1772 tcg_gen_movi_i64(t2
, 0x00ff00ff00ff00ffull
);
1773 tcg_gen_shri_i64(t0
, arg
, 8); /* t0 = .abcdefg */
1774 tcg_gen_and_i64(t1
, arg
, t2
); /* t1 = .b.d.f.h */
1775 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = .a.c.e.g */
1776 tcg_gen_shli_i64(t1
, t1
, 8); /* t1 = b.d.f.h. */
1777 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = badcfehg */
1779 tcg_gen_movi_i64(t2
, 0x0000ffff0000ffffull
);
1780 tcg_gen_shri_i64(t0
, ret
, 16); /* t0 = ..badcfe */
1781 tcg_gen_and_i64(t1
, ret
, t2
); /* t1 = ..dc..hg */
1782 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = ..ba..fe */
1783 tcg_gen_shli_i64(t1
, t1
, 16); /* t1 = dc..hg.. */
1784 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = dcbahgfe */
1786 tcg_gen_shri_i64(t0
, ret
, 32); /* t0 = ....dcba */
1787 tcg_gen_shli_i64(t1
, ret
, 32); /* t1 = hgfe.... */
1788 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = hgfedcba */
1790 tcg_temp_free_i64(t0
);
1791 tcg_temp_free_i64(t1
);
1792 tcg_temp_free_i64(t2
);
1796 void tcg_gen_not_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1798 if (TCG_TARGET_REG_BITS
== 32) {
1799 tcg_gen_not_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1800 tcg_gen_not_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
));
1801 } else if (TCG_TARGET_HAS_not_i64
) {
1802 tcg_gen_op2_i64(INDEX_op_not_i64
, ret
, arg
);
1804 tcg_gen_xori_i64(ret
, arg
, -1);
1808 void tcg_gen_andc_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1810 if (TCG_TARGET_REG_BITS
== 32) {
1811 tcg_gen_andc_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1812 tcg_gen_andc_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1813 } else if (TCG_TARGET_HAS_andc_i64
) {
1814 tcg_gen_op3_i64(INDEX_op_andc_i64
, ret
, arg1
, arg2
);
1816 TCGv_i64 t0
= tcg_temp_new_i64();
1817 tcg_gen_not_i64(t0
, arg2
);
1818 tcg_gen_and_i64(ret
, arg1
, t0
);
1819 tcg_temp_free_i64(t0
);
1823 void tcg_gen_eqv_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1825 if (TCG_TARGET_REG_BITS
== 32) {
1826 tcg_gen_eqv_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1827 tcg_gen_eqv_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1828 } else if (TCG_TARGET_HAS_eqv_i64
) {
1829 tcg_gen_op3_i64(INDEX_op_eqv_i64
, ret
, arg1
, arg2
);
1831 tcg_gen_xor_i64(ret
, arg1
, arg2
);
1832 tcg_gen_not_i64(ret
, ret
);
1836 void tcg_gen_nand_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1838 if (TCG_TARGET_REG_BITS
== 32) {
1839 tcg_gen_nand_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1840 tcg_gen_nand_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1841 } else if (TCG_TARGET_HAS_nand_i64
) {
1842 tcg_gen_op3_i64(INDEX_op_nand_i64
, ret
, arg1
, arg2
);
1844 tcg_gen_and_i64(ret
, arg1
, arg2
);
1845 tcg_gen_not_i64(ret
, ret
);
1849 void tcg_gen_nor_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1851 if (TCG_TARGET_REG_BITS
== 32) {
1852 tcg_gen_nor_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1853 tcg_gen_nor_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1854 } else if (TCG_TARGET_HAS_nor_i64
) {
1855 tcg_gen_op3_i64(INDEX_op_nor_i64
, ret
, arg1
, arg2
);
1857 tcg_gen_or_i64(ret
, arg1
, arg2
);
1858 tcg_gen_not_i64(ret
, ret
);
1862 void tcg_gen_orc_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1864 if (TCG_TARGET_REG_BITS
== 32) {
1865 tcg_gen_orc_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1866 tcg_gen_orc_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1867 } else if (TCG_TARGET_HAS_orc_i64
) {
1868 tcg_gen_op3_i64(INDEX_op_orc_i64
, ret
, arg1
, arg2
);
1870 TCGv_i64 t0
= tcg_temp_new_i64();
1871 tcg_gen_not_i64(t0
, arg2
);
1872 tcg_gen_or_i64(ret
, arg1
, t0
);
1873 tcg_temp_free_i64(t0
);
1877 void tcg_gen_clz_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1879 if (TCG_TARGET_HAS_clz_i64
) {
1880 tcg_gen_op3_i64(INDEX_op_clz_i64
, ret
, arg1
, arg2
);
1882 gen_helper_clz_i64(ret
, arg1
, arg2
);
1886 void tcg_gen_clzi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, uint64_t arg2
)
1888 if (TCG_TARGET_REG_BITS
== 32
1889 && TCG_TARGET_HAS_clz_i32
1890 && arg2
<= 0xffffffffu
) {
1891 TCGv_i32 t
= tcg_temp_new_i32();
1892 tcg_gen_clzi_i32(t
, TCGV_LOW(arg1
), arg2
- 32);
1893 tcg_gen_addi_i32(t
, t
, 32);
1894 tcg_gen_clz_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), t
);
1895 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1896 tcg_temp_free_i32(t
);
1898 TCGv_i64 t0
= tcg_const_i64(arg2
);
1899 tcg_gen_clz_i64(ret
, arg1
, t0
);
1900 tcg_temp_free_i64(t0
);
1904 void tcg_gen_ctz_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1906 if (TCG_TARGET_HAS_ctz_i64
) {
1907 tcg_gen_op3_i64(INDEX_op_ctz_i64
, ret
, arg1
, arg2
);
1908 } else if (TCG_TARGET_HAS_ctpop_i64
|| TCG_TARGET_HAS_clz_i64
) {
1909 TCGv_i64 z
, t
= tcg_temp_new_i64();
1911 if (TCG_TARGET_HAS_ctpop_i64
) {
1912 tcg_gen_subi_i64(t
, arg1
, 1);
1913 tcg_gen_andc_i64(t
, t
, arg1
);
1914 tcg_gen_ctpop_i64(t
, t
);
1916 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1917 tcg_gen_neg_i64(t
, arg1
);
1918 tcg_gen_and_i64(t
, t
, arg1
);
1919 tcg_gen_clzi_i64(t
, t
, 64);
1920 tcg_gen_xori_i64(t
, t
, 63);
1922 z
= tcg_constant_i64(0);
1923 tcg_gen_movcond_i64(TCG_COND_EQ
, ret
, arg1
, z
, arg2
, t
);
1924 tcg_temp_free_i64(t
);
1925 tcg_temp_free_i64(z
);
1927 gen_helper_ctz_i64(ret
, arg1
, arg2
);
1931 void tcg_gen_ctzi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, uint64_t arg2
)
1933 if (TCG_TARGET_REG_BITS
== 32
1934 && TCG_TARGET_HAS_ctz_i32
1935 && arg2
<= 0xffffffffu
) {
1936 TCGv_i32 t32
= tcg_temp_new_i32();
1937 tcg_gen_ctzi_i32(t32
, TCGV_HIGH(arg1
), arg2
- 32);
1938 tcg_gen_addi_i32(t32
, t32
, 32);
1939 tcg_gen_ctz_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), t32
);
1940 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1941 tcg_temp_free_i32(t32
);
1942 } else if (!TCG_TARGET_HAS_ctz_i64
1943 && TCG_TARGET_HAS_ctpop_i64
1945 /* This equivalence has the advantage of not requiring a fixup. */
1946 TCGv_i64 t
= tcg_temp_new_i64();
1947 tcg_gen_subi_i64(t
, arg1
, 1);
1948 tcg_gen_andc_i64(t
, t
, arg1
);
1949 tcg_gen_ctpop_i64(ret
, t
);
1950 tcg_temp_free_i64(t
);
1952 TCGv_i64 t0
= tcg_const_i64(arg2
);
1953 tcg_gen_ctz_i64(ret
, arg1
, t0
);
1954 tcg_temp_free_i64(t0
);
1958 void tcg_gen_clrsb_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1960 if (TCG_TARGET_HAS_clz_i64
|| TCG_TARGET_HAS_clz_i32
) {
1961 TCGv_i64 t
= tcg_temp_new_i64();
1962 tcg_gen_sari_i64(t
, arg
, 63);
1963 tcg_gen_xor_i64(t
, t
, arg
);
1964 tcg_gen_clzi_i64(t
, t
, 64);
1965 tcg_gen_subi_i64(ret
, t
, 1);
1966 tcg_temp_free_i64(t
);
1968 gen_helper_clrsb_i64(ret
, arg
);
1972 void tcg_gen_ctpop_i64(TCGv_i64 ret
, TCGv_i64 arg1
)
1974 if (TCG_TARGET_HAS_ctpop_i64
) {
1975 tcg_gen_op2_i64(INDEX_op_ctpop_i64
, ret
, arg1
);
1976 } else if (TCG_TARGET_REG_BITS
== 32 && TCG_TARGET_HAS_ctpop_i32
) {
1977 tcg_gen_ctpop_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
1978 tcg_gen_ctpop_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
1979 tcg_gen_add_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), TCGV_HIGH(ret
));
1980 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1982 gen_helper_ctpop_i64(ret
, arg1
);
1986 void tcg_gen_rotl_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1988 if (TCG_TARGET_HAS_rot_i64
) {
1989 tcg_gen_op3_i64(INDEX_op_rotl_i64
, ret
, arg1
, arg2
);
1992 t0
= tcg_temp_new_i64();
1993 t1
= tcg_temp_new_i64();
1994 tcg_gen_shl_i64(t0
, arg1
, arg2
);
1995 tcg_gen_subfi_i64(t1
, 64, arg2
);
1996 tcg_gen_shr_i64(t1
, arg1
, t1
);
1997 tcg_gen_or_i64(ret
, t0
, t1
);
1998 tcg_temp_free_i64(t0
);
1999 tcg_temp_free_i64(t1
);
2003 void tcg_gen_rotli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
2005 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
2006 /* some cases can be optimized here */
2008 tcg_gen_mov_i64(ret
, arg1
);
2009 } else if (TCG_TARGET_HAS_rot_i64
) {
2010 tcg_gen_rotl_i64(ret
, arg1
, tcg_constant_i64(arg2
));
2013 t0
= tcg_temp_new_i64();
2014 t1
= tcg_temp_new_i64();
2015 tcg_gen_shli_i64(t0
, arg1
, arg2
);
2016 tcg_gen_shri_i64(t1
, arg1
, 64 - arg2
);
2017 tcg_gen_or_i64(ret
, t0
, t1
);
2018 tcg_temp_free_i64(t0
);
2019 tcg_temp_free_i64(t1
);
2023 void tcg_gen_rotr_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2025 if (TCG_TARGET_HAS_rot_i64
) {
2026 tcg_gen_op3_i64(INDEX_op_rotr_i64
, ret
, arg1
, arg2
);
2029 t0
= tcg_temp_new_i64();
2030 t1
= tcg_temp_new_i64();
2031 tcg_gen_shr_i64(t0
, arg1
, arg2
);
2032 tcg_gen_subfi_i64(t1
, 64, arg2
);
2033 tcg_gen_shl_i64(t1
, arg1
, t1
);
2034 tcg_gen_or_i64(ret
, t0
, t1
);
2035 tcg_temp_free_i64(t0
);
2036 tcg_temp_free_i64(t1
);
2040 void tcg_gen_rotri_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
2042 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
2043 /* some cases can be optimized here */
2045 tcg_gen_mov_i64(ret
, arg1
);
2047 tcg_gen_rotli_i64(ret
, arg1
, 64 - arg2
);
2051 void tcg_gen_deposit_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
,
2052 unsigned int ofs
, unsigned int len
)
2057 tcg_debug_assert(ofs
< 64);
2058 tcg_debug_assert(len
> 0);
2059 tcg_debug_assert(len
<= 64);
2060 tcg_debug_assert(ofs
+ len
<= 64);
2063 tcg_gen_mov_i64(ret
, arg2
);
2066 if (TCG_TARGET_HAS_deposit_i64
&& TCG_TARGET_deposit_i64_valid(ofs
, len
)) {
2067 tcg_gen_op5ii_i64(INDEX_op_deposit_i64
, ret
, arg1
, arg2
, ofs
, len
);
2071 if (TCG_TARGET_REG_BITS
== 32) {
2073 tcg_gen_deposit_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
),
2074 TCGV_LOW(arg2
), ofs
- 32, len
);
2075 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
2078 if (ofs
+ len
<= 32) {
2079 tcg_gen_deposit_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
),
2080 TCGV_LOW(arg2
), ofs
, len
);
2081 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
2086 t1
= tcg_temp_new_i64();
2088 if (TCG_TARGET_HAS_extract2_i64
) {
2089 if (ofs
+ len
== 64) {
2090 tcg_gen_shli_i64(t1
, arg1
, len
);
2091 tcg_gen_extract2_i64(ret
, t1
, arg2
, len
);
2095 tcg_gen_extract2_i64(ret
, arg1
, arg2
, len
);
2096 tcg_gen_rotli_i64(ret
, ret
, len
);
2101 mask
= (1ull << len
) - 1;
2102 if (ofs
+ len
< 64) {
2103 tcg_gen_andi_i64(t1
, arg2
, mask
);
2104 tcg_gen_shli_i64(t1
, t1
, ofs
);
2106 tcg_gen_shli_i64(t1
, arg2
, ofs
);
2108 tcg_gen_andi_i64(ret
, arg1
, ~(mask
<< ofs
));
2109 tcg_gen_or_i64(ret
, ret
, t1
);
2111 tcg_temp_free_i64(t1
);
2114 void tcg_gen_deposit_z_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2115 unsigned int ofs
, unsigned int len
)
2117 tcg_debug_assert(ofs
< 64);
2118 tcg_debug_assert(len
> 0);
2119 tcg_debug_assert(len
<= 64);
2120 tcg_debug_assert(ofs
+ len
<= 64);
2122 if (ofs
+ len
== 64) {
2123 tcg_gen_shli_i64(ret
, arg
, ofs
);
2124 } else if (ofs
== 0) {
2125 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2126 } else if (TCG_TARGET_HAS_deposit_i64
2127 && TCG_TARGET_deposit_i64_valid(ofs
, len
)) {
2128 TCGv_i64 zero
= tcg_constant_i64(0);
2129 tcg_gen_op5ii_i64(INDEX_op_deposit_i64
, ret
, zero
, arg
, ofs
, len
);
2131 if (TCG_TARGET_REG_BITS
== 32) {
2133 tcg_gen_deposit_z_i32(TCGV_HIGH(ret
), TCGV_LOW(arg
),
2135 tcg_gen_movi_i32(TCGV_LOW(ret
), 0);
2138 if (ofs
+ len
<= 32) {
2139 tcg_gen_deposit_z_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2140 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2144 /* To help two-operand hosts we prefer to zero-extend first,
2145 which allows ARG to stay live. */
2148 if (TCG_TARGET_HAS_ext32u_i64
) {
2149 tcg_gen_ext32u_i64(ret
, arg
);
2150 tcg_gen_shli_i64(ret
, ret
, ofs
);
2155 if (TCG_TARGET_HAS_ext16u_i64
) {
2156 tcg_gen_ext16u_i64(ret
, arg
);
2157 tcg_gen_shli_i64(ret
, ret
, ofs
);
2162 if (TCG_TARGET_HAS_ext8u_i64
) {
2163 tcg_gen_ext8u_i64(ret
, arg
);
2164 tcg_gen_shli_i64(ret
, ret
, ofs
);
2169 /* Otherwise prefer zero-extension over AND for code size. */
2170 switch (ofs
+ len
) {
2172 if (TCG_TARGET_HAS_ext32u_i64
) {
2173 tcg_gen_shli_i64(ret
, arg
, ofs
);
2174 tcg_gen_ext32u_i64(ret
, ret
);
2179 if (TCG_TARGET_HAS_ext16u_i64
) {
2180 tcg_gen_shli_i64(ret
, arg
, ofs
);
2181 tcg_gen_ext16u_i64(ret
, ret
);
2186 if (TCG_TARGET_HAS_ext8u_i64
) {
2187 tcg_gen_shli_i64(ret
, arg
, ofs
);
2188 tcg_gen_ext8u_i64(ret
, ret
);
2193 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2194 tcg_gen_shli_i64(ret
, ret
, ofs
);
2198 void tcg_gen_extract_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2199 unsigned int ofs
, unsigned int len
)
2201 tcg_debug_assert(ofs
< 64);
2202 tcg_debug_assert(len
> 0);
2203 tcg_debug_assert(len
<= 64);
2204 tcg_debug_assert(ofs
+ len
<= 64);
2206 /* Canonicalize certain special cases, even if extract is supported. */
2207 if (ofs
+ len
== 64) {
2208 tcg_gen_shri_i64(ret
, arg
, 64 - len
);
2212 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2216 if (TCG_TARGET_REG_BITS
== 32) {
2217 /* Look for a 32-bit extract within one of the two words. */
2219 tcg_gen_extract_i32(TCGV_LOW(ret
), TCGV_HIGH(arg
), ofs
- 32, len
);
2220 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2223 if (ofs
+ len
<= 32) {
2224 tcg_gen_extract_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2225 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2228 /* The field is split across two words. One double-word
2229 shift is better than two double-word shifts. */
2233 if (TCG_TARGET_HAS_extract_i64
2234 && TCG_TARGET_extract_i64_valid(ofs
, len
)) {
2235 tcg_gen_op4ii_i64(INDEX_op_extract_i64
, ret
, arg
, ofs
, len
);
2239 /* Assume that zero-extension, if available, is cheaper than a shift. */
2240 switch (ofs
+ len
) {
2242 if (TCG_TARGET_HAS_ext32u_i64
) {
2243 tcg_gen_ext32u_i64(ret
, arg
);
2244 tcg_gen_shri_i64(ret
, ret
, ofs
);
2249 if (TCG_TARGET_HAS_ext16u_i64
) {
2250 tcg_gen_ext16u_i64(ret
, arg
);
2251 tcg_gen_shri_i64(ret
, ret
, ofs
);
2256 if (TCG_TARGET_HAS_ext8u_i64
) {
2257 tcg_gen_ext8u_i64(ret
, arg
);
2258 tcg_gen_shri_i64(ret
, ret
, ofs
);
2264 /* ??? Ideally we'd know what values are available for immediate AND.
2265 Assume that 8 bits are available, plus the special cases of 16 and 32,
2266 so that we get ext8u, ext16u, and ext32u. */
2268 case 1 ... 8: case 16: case 32:
2270 tcg_gen_shri_i64(ret
, arg
, ofs
);
2271 tcg_gen_andi_i64(ret
, ret
, (1ull << len
) - 1);
2274 tcg_gen_shli_i64(ret
, arg
, 64 - len
- ofs
);
2275 tcg_gen_shri_i64(ret
, ret
, 64 - len
);
2280 void tcg_gen_sextract_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2281 unsigned int ofs
, unsigned int len
)
2283 tcg_debug_assert(ofs
< 64);
2284 tcg_debug_assert(len
> 0);
2285 tcg_debug_assert(len
<= 64);
2286 tcg_debug_assert(ofs
+ len
<= 64);
2288 /* Canonicalize certain special cases, even if sextract is supported. */
2289 if (ofs
+ len
== 64) {
2290 tcg_gen_sari_i64(ret
, arg
, 64 - len
);
2296 tcg_gen_ext32s_i64(ret
, arg
);
2299 tcg_gen_ext16s_i64(ret
, arg
);
2302 tcg_gen_ext8s_i64(ret
, arg
);
2307 if (TCG_TARGET_REG_BITS
== 32) {
2308 /* Look for a 32-bit extract within one of the two words. */
2310 tcg_gen_sextract_i32(TCGV_LOW(ret
), TCGV_HIGH(arg
), ofs
- 32, len
);
2311 } else if (ofs
+ len
<= 32) {
2312 tcg_gen_sextract_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2313 } else if (ofs
== 0) {
2314 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
2315 tcg_gen_sextract_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
), 0, len
- 32);
2317 } else if (len
> 32) {
2318 TCGv_i32 t
= tcg_temp_new_i32();
2319 /* Extract the bits for the high word normally. */
2320 tcg_gen_sextract_i32(t
, TCGV_HIGH(arg
), ofs
+ 32, len
- 32);
2321 /* Shift the field down for the low part. */
2322 tcg_gen_shri_i64(ret
, arg
, ofs
);
2323 /* Overwrite the shift into the high part. */
2324 tcg_gen_mov_i32(TCGV_HIGH(ret
), t
);
2325 tcg_temp_free_i32(t
);
2328 /* Shift the field down for the low part, such that the
2329 field sits at the MSB. */
2330 tcg_gen_shri_i64(ret
, arg
, ofs
+ len
- 32);
2331 /* Shift the field down from the MSB, sign extending. */
2332 tcg_gen_sari_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), 32 - len
);
2334 /* Sign-extend the field from 32 bits. */
2335 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
2339 if (TCG_TARGET_HAS_sextract_i64
2340 && TCG_TARGET_extract_i64_valid(ofs
, len
)) {
2341 tcg_gen_op4ii_i64(INDEX_op_sextract_i64
, ret
, arg
, ofs
, len
);
2345 /* Assume that sign-extension, if available, is cheaper than a shift. */
2346 switch (ofs
+ len
) {
2348 if (TCG_TARGET_HAS_ext32s_i64
) {
2349 tcg_gen_ext32s_i64(ret
, arg
);
2350 tcg_gen_sari_i64(ret
, ret
, ofs
);
2355 if (TCG_TARGET_HAS_ext16s_i64
) {
2356 tcg_gen_ext16s_i64(ret
, arg
);
2357 tcg_gen_sari_i64(ret
, ret
, ofs
);
2362 if (TCG_TARGET_HAS_ext8s_i64
) {
2363 tcg_gen_ext8s_i64(ret
, arg
);
2364 tcg_gen_sari_i64(ret
, ret
, ofs
);
2371 if (TCG_TARGET_HAS_ext32s_i64
) {
2372 tcg_gen_shri_i64(ret
, arg
, ofs
);
2373 tcg_gen_ext32s_i64(ret
, ret
);
2378 if (TCG_TARGET_HAS_ext16s_i64
) {
2379 tcg_gen_shri_i64(ret
, arg
, ofs
);
2380 tcg_gen_ext16s_i64(ret
, ret
);
2385 if (TCG_TARGET_HAS_ext8s_i64
) {
2386 tcg_gen_shri_i64(ret
, arg
, ofs
);
2387 tcg_gen_ext8s_i64(ret
, ret
);
2392 tcg_gen_shli_i64(ret
, arg
, 64 - len
- ofs
);
2393 tcg_gen_sari_i64(ret
, ret
, 64 - len
);
2397 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2398 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2400 void tcg_gen_extract2_i64(TCGv_i64 ret
, TCGv_i64 al
, TCGv_i64 ah
,
2403 tcg_debug_assert(ofs
<= 64);
2405 tcg_gen_mov_i64(ret
, al
);
2406 } else if (ofs
== 64) {
2407 tcg_gen_mov_i64(ret
, ah
);
2408 } else if (al
== ah
) {
2409 tcg_gen_rotri_i64(ret
, al
, ofs
);
2410 } else if (TCG_TARGET_HAS_extract2_i64
) {
2411 tcg_gen_op4i_i64(INDEX_op_extract2_i64
, ret
, al
, ah
, ofs
);
2413 TCGv_i64 t0
= tcg_temp_new_i64();
2414 tcg_gen_shri_i64(t0
, al
, ofs
);
2415 tcg_gen_deposit_i64(ret
, t0
, ah
, 64 - ofs
, ofs
);
2416 tcg_temp_free_i64(t0
);
2420 void tcg_gen_movcond_i64(TCGCond cond
, TCGv_i64 ret
, TCGv_i64 c1
,
2421 TCGv_i64 c2
, TCGv_i64 v1
, TCGv_i64 v2
)
2423 if (cond
== TCG_COND_ALWAYS
) {
2424 tcg_gen_mov_i64(ret
, v1
);
2425 } else if (cond
== TCG_COND_NEVER
) {
2426 tcg_gen_mov_i64(ret
, v2
);
2427 } else if (TCG_TARGET_REG_BITS
== 32) {
2428 TCGv_i32 t0
= tcg_temp_new_i32();
2429 TCGv_i32 t1
= tcg_temp_new_i32();
2430 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, t0
,
2431 TCGV_LOW(c1
), TCGV_HIGH(c1
),
2432 TCGV_LOW(c2
), TCGV_HIGH(c2
), cond
);
2434 if (TCG_TARGET_HAS_movcond_i32
) {
2435 tcg_gen_movi_i32(t1
, 0);
2436 tcg_gen_movcond_i32(TCG_COND_NE
, TCGV_LOW(ret
), t0
, t1
,
2437 TCGV_LOW(v1
), TCGV_LOW(v2
));
2438 tcg_gen_movcond_i32(TCG_COND_NE
, TCGV_HIGH(ret
), t0
, t1
,
2439 TCGV_HIGH(v1
), TCGV_HIGH(v2
));
2441 tcg_gen_neg_i32(t0
, t0
);
2443 tcg_gen_and_i32(t1
, TCGV_LOW(v1
), t0
);
2444 tcg_gen_andc_i32(TCGV_LOW(ret
), TCGV_LOW(v2
), t0
);
2445 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), t1
);
2447 tcg_gen_and_i32(t1
, TCGV_HIGH(v1
), t0
);
2448 tcg_gen_andc_i32(TCGV_HIGH(ret
), TCGV_HIGH(v2
), t0
);
2449 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(ret
), t1
);
2451 tcg_temp_free_i32(t0
);
2452 tcg_temp_free_i32(t1
);
2453 } else if (TCG_TARGET_HAS_movcond_i64
) {
2454 tcg_gen_op6i_i64(INDEX_op_movcond_i64
, ret
, c1
, c2
, v1
, v2
, cond
);
2456 TCGv_i64 t0
= tcg_temp_new_i64();
2457 TCGv_i64 t1
= tcg_temp_new_i64();
2458 tcg_gen_setcond_i64(cond
, t0
, c1
, c2
);
2459 tcg_gen_neg_i64(t0
, t0
);
2460 tcg_gen_and_i64(t1
, v1
, t0
);
2461 tcg_gen_andc_i64(ret
, v2
, t0
);
2462 tcg_gen_or_i64(ret
, ret
, t1
);
2463 tcg_temp_free_i64(t0
);
2464 tcg_temp_free_i64(t1
);
2468 void tcg_gen_add2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 al
,
2469 TCGv_i64 ah
, TCGv_i64 bl
, TCGv_i64 bh
)
2471 if (TCG_TARGET_HAS_add2_i64
) {
2472 tcg_gen_op6_i64(INDEX_op_add2_i64
, rl
, rh
, al
, ah
, bl
, bh
);
2474 TCGv_i64 t0
= tcg_temp_new_i64();
2475 TCGv_i64 t1
= tcg_temp_new_i64();
2476 tcg_gen_add_i64(t0
, al
, bl
);
2477 tcg_gen_setcond_i64(TCG_COND_LTU
, t1
, t0
, al
);
2478 tcg_gen_add_i64(rh
, ah
, bh
);
2479 tcg_gen_add_i64(rh
, rh
, t1
);
2480 tcg_gen_mov_i64(rl
, t0
);
2481 tcg_temp_free_i64(t0
);
2482 tcg_temp_free_i64(t1
);
2486 void tcg_gen_sub2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 al
,
2487 TCGv_i64 ah
, TCGv_i64 bl
, TCGv_i64 bh
)
2489 if (TCG_TARGET_HAS_sub2_i64
) {
2490 tcg_gen_op6_i64(INDEX_op_sub2_i64
, rl
, rh
, al
, ah
, bl
, bh
);
2492 TCGv_i64 t0
= tcg_temp_new_i64();
2493 TCGv_i64 t1
= tcg_temp_new_i64();
2494 tcg_gen_sub_i64(t0
, al
, bl
);
2495 tcg_gen_setcond_i64(TCG_COND_LTU
, t1
, al
, bl
);
2496 tcg_gen_sub_i64(rh
, ah
, bh
);
2497 tcg_gen_sub_i64(rh
, rh
, t1
);
2498 tcg_gen_mov_i64(rl
, t0
);
2499 tcg_temp_free_i64(t0
);
2500 tcg_temp_free_i64(t1
);
2504 void tcg_gen_mulu2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2506 if (TCG_TARGET_HAS_mulu2_i64
) {
2507 tcg_gen_op4_i64(INDEX_op_mulu2_i64
, rl
, rh
, arg1
, arg2
);
2508 } else if (TCG_TARGET_HAS_muluh_i64
) {
2509 TCGv_i64 t
= tcg_temp_new_i64();
2510 tcg_gen_op3_i64(INDEX_op_mul_i64
, t
, arg1
, arg2
);
2511 tcg_gen_op3_i64(INDEX_op_muluh_i64
, rh
, arg1
, arg2
);
2512 tcg_gen_mov_i64(rl
, t
);
2513 tcg_temp_free_i64(t
);
2515 TCGv_i64 t0
= tcg_temp_new_i64();
2516 tcg_gen_mul_i64(t0
, arg1
, arg2
);
2517 gen_helper_muluh_i64(rh
, arg1
, arg2
);
2518 tcg_gen_mov_i64(rl
, t0
);
2519 tcg_temp_free_i64(t0
);
2523 void tcg_gen_muls2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2525 if (TCG_TARGET_HAS_muls2_i64
) {
2526 tcg_gen_op4_i64(INDEX_op_muls2_i64
, rl
, rh
, arg1
, arg2
);
2527 } else if (TCG_TARGET_HAS_mulsh_i64
) {
2528 TCGv_i64 t
= tcg_temp_new_i64();
2529 tcg_gen_op3_i64(INDEX_op_mul_i64
, t
, arg1
, arg2
);
2530 tcg_gen_op3_i64(INDEX_op_mulsh_i64
, rh
, arg1
, arg2
);
2531 tcg_gen_mov_i64(rl
, t
);
2532 tcg_temp_free_i64(t
);
2533 } else if (TCG_TARGET_HAS_mulu2_i64
|| TCG_TARGET_HAS_muluh_i64
) {
2534 TCGv_i64 t0
= tcg_temp_new_i64();
2535 TCGv_i64 t1
= tcg_temp_new_i64();
2536 TCGv_i64 t2
= tcg_temp_new_i64();
2537 TCGv_i64 t3
= tcg_temp_new_i64();
2538 tcg_gen_mulu2_i64(t0
, t1
, arg1
, arg2
);
2539 /* Adjust for negative inputs. */
2540 tcg_gen_sari_i64(t2
, arg1
, 63);
2541 tcg_gen_sari_i64(t3
, arg2
, 63);
2542 tcg_gen_and_i64(t2
, t2
, arg2
);
2543 tcg_gen_and_i64(t3
, t3
, arg1
);
2544 tcg_gen_sub_i64(rh
, t1
, t2
);
2545 tcg_gen_sub_i64(rh
, rh
, t3
);
2546 tcg_gen_mov_i64(rl
, t0
);
2547 tcg_temp_free_i64(t0
);
2548 tcg_temp_free_i64(t1
);
2549 tcg_temp_free_i64(t2
);
2550 tcg_temp_free_i64(t3
);
2552 TCGv_i64 t0
= tcg_temp_new_i64();
2553 tcg_gen_mul_i64(t0
, arg1
, arg2
);
2554 gen_helper_mulsh_i64(rh
, arg1
, arg2
);
2555 tcg_gen_mov_i64(rl
, t0
);
2556 tcg_temp_free_i64(t0
);
2560 void tcg_gen_mulsu2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2562 TCGv_i64 t0
= tcg_temp_new_i64();
2563 TCGv_i64 t1
= tcg_temp_new_i64();
2564 TCGv_i64 t2
= tcg_temp_new_i64();
2565 tcg_gen_mulu2_i64(t0
, t1
, arg1
, arg2
);
2566 /* Adjust for negative input for the signed arg1. */
2567 tcg_gen_sari_i64(t2
, arg1
, 63);
2568 tcg_gen_and_i64(t2
, t2
, arg2
);
2569 tcg_gen_sub_i64(rh
, t1
, t2
);
2570 tcg_gen_mov_i64(rl
, t0
);
2571 tcg_temp_free_i64(t0
);
2572 tcg_temp_free_i64(t1
);
2573 tcg_temp_free_i64(t2
);
2576 void tcg_gen_smin_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2578 tcg_gen_movcond_i64(TCG_COND_LT
, ret
, a
, b
, a
, b
);
2581 void tcg_gen_umin_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2583 tcg_gen_movcond_i64(TCG_COND_LTU
, ret
, a
, b
, a
, b
);
2586 void tcg_gen_smax_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2588 tcg_gen_movcond_i64(TCG_COND_LT
, ret
, a
, b
, b
, a
);
2591 void tcg_gen_umax_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2593 tcg_gen_movcond_i64(TCG_COND_LTU
, ret
, a
, b
, b
, a
);
2596 void tcg_gen_abs_i64(TCGv_i64 ret
, TCGv_i64 a
)
2598 TCGv_i64 t
= tcg_temp_new_i64();
2600 tcg_gen_sari_i64(t
, a
, 63);
2601 tcg_gen_xor_i64(ret
, a
, t
);
2602 tcg_gen_sub_i64(ret
, ret
, t
);
2603 tcg_temp_free_i64(t
);
2606 /* Size changing operations. */
2608 void tcg_gen_extrl_i64_i32(TCGv_i32 ret
, TCGv_i64 arg
)
2610 if (TCG_TARGET_REG_BITS
== 32) {
2611 tcg_gen_mov_i32(ret
, TCGV_LOW(arg
));
2612 } else if (TCG_TARGET_HAS_extrl_i64_i32
) {
2613 tcg_gen_op2(INDEX_op_extrl_i64_i32
,
2614 tcgv_i32_arg(ret
), tcgv_i64_arg(arg
));
2616 tcg_gen_mov_i32(ret
, (TCGv_i32
)arg
);
2620 void tcg_gen_extrh_i64_i32(TCGv_i32 ret
, TCGv_i64 arg
)
2622 if (TCG_TARGET_REG_BITS
== 32) {
2623 tcg_gen_mov_i32(ret
, TCGV_HIGH(arg
));
2624 } else if (TCG_TARGET_HAS_extrh_i64_i32
) {
2625 tcg_gen_op2(INDEX_op_extrh_i64_i32
,
2626 tcgv_i32_arg(ret
), tcgv_i64_arg(arg
));
2628 TCGv_i64 t
= tcg_temp_new_i64();
2629 tcg_gen_shri_i64(t
, arg
, 32);
2630 tcg_gen_mov_i32(ret
, (TCGv_i32
)t
);
2631 tcg_temp_free_i64(t
);
2635 void tcg_gen_extu_i32_i64(TCGv_i64 ret
, TCGv_i32 arg
)
2637 if (TCG_TARGET_REG_BITS
== 32) {
2638 tcg_gen_mov_i32(TCGV_LOW(ret
), arg
);
2639 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2641 tcg_gen_op2(INDEX_op_extu_i32_i64
,
2642 tcgv_i64_arg(ret
), tcgv_i32_arg(arg
));
2646 void tcg_gen_ext_i32_i64(TCGv_i64 ret
, TCGv_i32 arg
)
2648 if (TCG_TARGET_REG_BITS
== 32) {
2649 tcg_gen_mov_i32(TCGV_LOW(ret
), arg
);
2650 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
2652 tcg_gen_op2(INDEX_op_ext_i32_i64
,
2653 tcgv_i64_arg(ret
), tcgv_i32_arg(arg
));
2657 void tcg_gen_concat_i32_i64(TCGv_i64 dest
, TCGv_i32 low
, TCGv_i32 high
)
2661 if (TCG_TARGET_REG_BITS
== 32) {
2662 tcg_gen_mov_i32(TCGV_LOW(dest
), low
);
2663 tcg_gen_mov_i32(TCGV_HIGH(dest
), high
);
2667 tmp
= tcg_temp_new_i64();
2668 /* These extensions are only needed for type correctness.
2669 We may be able to do better given target specific information. */
2670 tcg_gen_extu_i32_i64(tmp
, high
);
2671 tcg_gen_extu_i32_i64(dest
, low
);
2672 /* If deposit is available, use it. Otherwise use the extra
2673 knowledge that we have of the zero-extensions above. */
2674 if (TCG_TARGET_HAS_deposit_i64
&& TCG_TARGET_deposit_i64_valid(32, 32)) {
2675 tcg_gen_deposit_i64(dest
, dest
, tmp
, 32, 32);
2677 tcg_gen_shli_i64(tmp
, tmp
, 32);
2678 tcg_gen_or_i64(dest
, dest
, tmp
);
2680 tcg_temp_free_i64(tmp
);
2683 void tcg_gen_extr_i64_i32(TCGv_i32 lo
, TCGv_i32 hi
, TCGv_i64 arg
)
2685 if (TCG_TARGET_REG_BITS
== 32) {
2686 tcg_gen_mov_i32(lo
, TCGV_LOW(arg
));
2687 tcg_gen_mov_i32(hi
, TCGV_HIGH(arg
));
2689 tcg_gen_extrl_i64_i32(lo
, arg
);
2690 tcg_gen_extrh_i64_i32(hi
, arg
);
2694 void tcg_gen_extr32_i64(TCGv_i64 lo
, TCGv_i64 hi
, TCGv_i64 arg
)
2696 tcg_gen_ext32u_i64(lo
, arg
);
2697 tcg_gen_shri_i64(hi
, arg
, 32);
2700 /* QEMU specific operations. */
2702 void tcg_gen_exit_tb(const TranslationBlock
*tb
, unsigned idx
)
2705 * Let the jit code return the read-only version of the
2706 * TranslationBlock, so that we minimize the pc-relative
2707 * distance of the address of the exit_tb code to TB.
2708 * This will improve utilization of pc-relative address loads.
2710 * TODO: Move this to translator_loop, so that all const
2711 * TranslationBlock pointers refer to read-only memory.
2712 * This requires coordination with targets that do not use
2713 * the translator_loop.
2715 uintptr_t val
= (uintptr_t)tcg_splitwx_to_rx((void *)tb
) + idx
;
2718 tcg_debug_assert(idx
== 0);
2719 } else if (idx
<= TB_EXIT_IDXMAX
) {
2720 #ifdef CONFIG_DEBUG_TCG
2721 /* This is an exit following a goto_tb. Verify that we have
2722 seen this numbered exit before, via tcg_gen_goto_tb. */
2723 tcg_debug_assert(tcg_ctx
->goto_tb_issue_mask
& (1 << idx
));
2726 /* This is an exit via the exitreq label. */
2727 tcg_debug_assert(idx
== TB_EXIT_REQUESTED
);
2730 plugin_gen_disable_mem_helpers();
2731 tcg_gen_op1i(INDEX_op_exit_tb
, val
);
2734 void tcg_gen_goto_tb(unsigned idx
)
2736 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
2737 tcg_debug_assert(!(tcg_ctx
->tb_cflags
& CF_NO_GOTO_TB
));
2738 /* We only support two chained exits. */
2739 tcg_debug_assert(idx
<= TB_EXIT_IDXMAX
);
2740 #ifdef CONFIG_DEBUG_TCG
2741 /* Verify that we haven't seen this numbered exit before. */
2742 tcg_debug_assert((tcg_ctx
->goto_tb_issue_mask
& (1 << idx
)) == 0);
2743 tcg_ctx
->goto_tb_issue_mask
|= 1 << idx
;
2745 plugin_gen_disable_mem_helpers();
2746 tcg_gen_op1i(INDEX_op_goto_tb
, idx
);
2749 void tcg_gen_lookup_and_goto_ptr(void)
2753 if (tcg_ctx
->tb_cflags
& CF_NO_GOTO_PTR
) {
2754 tcg_gen_exit_tb(NULL
, 0);
2758 plugin_gen_disable_mem_helpers();
2759 ptr
= tcg_temp_new_ptr();
2760 gen_helper_lookup_tb_ptr(ptr
, cpu_env
);
2761 tcg_gen_op1i(INDEX_op_goto_ptr
, tcgv_ptr_arg(ptr
));
2762 tcg_temp_free_ptr(ptr
);
2765 static inline MemOp
tcg_canonicalize_memop(MemOp op
, bool is64
, bool st
)
2767 /* Trigger the asserts within as early as possible. */
2768 unsigned a_bits
= get_alignment_bits(op
);
2770 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
2771 if (a_bits
== (op
& MO_SIZE
)) {
2772 op
= (op
& ~MO_AMASK
) | MO_ALIGN
;
2775 switch (op
& MO_SIZE
) {
2793 g_assert_not_reached();
2801 static void gen_ldst_i32(TCGOpcode opc
, TCGv_i32 val
, TCGv addr
,
2802 MemOp memop
, TCGArg idx
)
2804 MemOpIdx oi
= make_memop_idx(memop
, idx
);
2805 #if TARGET_LONG_BITS == 32
2806 tcg_gen_op3i_i32(opc
, val
, addr
, oi
);
2808 if (TCG_TARGET_REG_BITS
== 32) {
2809 tcg_gen_op4i_i32(opc
, val
, TCGV_LOW(addr
), TCGV_HIGH(addr
), oi
);
2811 tcg_gen_op3(opc
, tcgv_i32_arg(val
), tcgv_i64_arg(addr
), oi
);
2816 static void gen_ldst_i64(TCGOpcode opc
, TCGv_i64 val
, TCGv addr
,
2817 MemOp memop
, TCGArg idx
)
2819 MemOpIdx oi
= make_memop_idx(memop
, idx
);
2820 #if TARGET_LONG_BITS == 32
2821 if (TCG_TARGET_REG_BITS
== 32) {
2822 tcg_gen_op4i_i32(opc
, TCGV_LOW(val
), TCGV_HIGH(val
), addr
, oi
);
2824 tcg_gen_op3(opc
, tcgv_i64_arg(val
), tcgv_i32_arg(addr
), oi
);
2827 if (TCG_TARGET_REG_BITS
== 32) {
2828 tcg_gen_op5i_i32(opc
, TCGV_LOW(val
), TCGV_HIGH(val
),
2829 TCGV_LOW(addr
), TCGV_HIGH(addr
), oi
);
2831 tcg_gen_op3i_i64(opc
, val
, addr
, oi
);
2836 static void tcg_gen_req_mo(TCGBar type
)
2838 #ifdef TCG_GUEST_DEFAULT_MO
2839 type
&= TCG_GUEST_DEFAULT_MO
;
2841 type
&= ~TCG_TARGET_DEFAULT_MO
;
2843 tcg_gen_mb(type
| TCG_BAR_SC
);
2847 static inline TCGv
plugin_prep_mem_callbacks(TCGv vaddr
)
2849 #ifdef CONFIG_PLUGIN
2850 if (tcg_ctx
->plugin_insn
!= NULL
) {
2851 /* Save a copy of the vaddr for use after a load. */
2852 TCGv temp
= tcg_temp_new();
2853 tcg_gen_mov_tl(temp
, vaddr
);
2860 static void plugin_gen_mem_callbacks(TCGv vaddr
, MemOpIdx oi
,
2861 enum qemu_plugin_mem_rw rw
)
2863 #ifdef CONFIG_PLUGIN
2864 if (tcg_ctx
->plugin_insn
!= NULL
) {
2865 qemu_plugin_meminfo_t info
= make_plugin_meminfo(oi
, rw
);
2866 plugin_gen_empty_mem_callback(vaddr
, info
);
2867 tcg_temp_free(vaddr
);
2872 void tcg_gen_qemu_ld_i32(TCGv_i32 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2877 tcg_gen_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2878 memop
= tcg_canonicalize_memop(memop
, 0, 0);
2879 oi
= make_memop_idx(memop
, idx
);
2880 trace_guest_ld_before_tcg(tcg_ctx
->cpu
, cpu_env
, addr
, oi
);
2883 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2885 /* The bswap primitive benefits from zero-extended input. */
2886 if ((memop
& MO_SSIZE
) == MO_SW
) {
2891 addr
= plugin_prep_mem_callbacks(addr
);
2892 gen_ldst_i32(INDEX_op_qemu_ld_i32
, val
, addr
, memop
, idx
);
2893 plugin_gen_mem_callbacks(addr
, oi
, QEMU_PLUGIN_MEM_R
);
2895 if ((orig_memop
^ memop
) & MO_BSWAP
) {
2896 switch (orig_memop
& MO_SIZE
) {
2898 tcg_gen_bswap16_i32(val
, val
, (orig_memop
& MO_SIGN
2899 ? TCG_BSWAP_IZ
| TCG_BSWAP_OS
2900 : TCG_BSWAP_IZ
| TCG_BSWAP_OZ
));
2903 tcg_gen_bswap32_i32(val
, val
);
2906 g_assert_not_reached();
2911 void tcg_gen_qemu_st_i32(TCGv_i32 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2913 TCGv_i32 swap
= NULL
;
2916 tcg_gen_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2917 memop
= tcg_canonicalize_memop(memop
, 0, 1);
2918 oi
= make_memop_idx(memop
, idx
);
2919 trace_guest_st_before_tcg(tcg_ctx
->cpu
, cpu_env
, addr
, oi
);
2921 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2922 swap
= tcg_temp_new_i32();
2923 switch (memop
& MO_SIZE
) {
2925 tcg_gen_bswap16_i32(swap
, val
, 0);
2928 tcg_gen_bswap32_i32(swap
, val
);
2931 g_assert_not_reached();
2937 addr
= plugin_prep_mem_callbacks(addr
);
2938 if (TCG_TARGET_HAS_qemu_st8_i32
&& (memop
& MO_SIZE
) == MO_8
) {
2939 gen_ldst_i32(INDEX_op_qemu_st8_i32
, val
, addr
, memop
, idx
);
2941 gen_ldst_i32(INDEX_op_qemu_st_i32
, val
, addr
, memop
, idx
);
2943 plugin_gen_mem_callbacks(addr
, oi
, QEMU_PLUGIN_MEM_W
);
2946 tcg_temp_free_i32(swap
);
2950 void tcg_gen_qemu_ld_i64(TCGv_i64 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2955 if (TCG_TARGET_REG_BITS
== 32 && (memop
& MO_SIZE
) < MO_64
) {
2956 tcg_gen_qemu_ld_i32(TCGV_LOW(val
), addr
, idx
, memop
);
2957 if (memop
& MO_SIGN
) {
2958 tcg_gen_sari_i32(TCGV_HIGH(val
), TCGV_LOW(val
), 31);
2960 tcg_gen_movi_i32(TCGV_HIGH(val
), 0);
2965 tcg_gen_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2966 memop
= tcg_canonicalize_memop(memop
, 1, 0);
2967 oi
= make_memop_idx(memop
, idx
);
2968 trace_guest_ld_before_tcg(tcg_ctx
->cpu
, cpu_env
, addr
, oi
);
2971 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2973 /* The bswap primitive benefits from zero-extended input. */
2974 if ((memop
& MO_SIGN
) && (memop
& MO_SIZE
) < MO_64
) {
2979 addr
= plugin_prep_mem_callbacks(addr
);
2980 gen_ldst_i64(INDEX_op_qemu_ld_i64
, val
, addr
, memop
, idx
);
2981 plugin_gen_mem_callbacks(addr
, oi
, QEMU_PLUGIN_MEM_R
);
2983 if ((orig_memop
^ memop
) & MO_BSWAP
) {
2984 int flags
= (orig_memop
& MO_SIGN
2985 ? TCG_BSWAP_IZ
| TCG_BSWAP_OS
2986 : TCG_BSWAP_IZ
| TCG_BSWAP_OZ
);
2987 switch (orig_memop
& MO_SIZE
) {
2989 tcg_gen_bswap16_i64(val
, val
, flags
);
2992 tcg_gen_bswap32_i64(val
, val
, flags
);
2995 tcg_gen_bswap64_i64(val
, val
);
2998 g_assert_not_reached();
3003 void tcg_gen_qemu_st_i64(TCGv_i64 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
3005 TCGv_i64 swap
= NULL
;
3008 if (TCG_TARGET_REG_BITS
== 32 && (memop
& MO_SIZE
) < MO_64
) {
3009 tcg_gen_qemu_st_i32(TCGV_LOW(val
), addr
, idx
, memop
);
3013 tcg_gen_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
3014 memop
= tcg_canonicalize_memop(memop
, 1, 1);
3015 oi
= make_memop_idx(memop
, idx
);
3016 trace_guest_st_before_tcg(tcg_ctx
->cpu
, cpu_env
, addr
, oi
);
3018 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
3019 swap
= tcg_temp_new_i64();
3020 switch (memop
& MO_SIZE
) {
3022 tcg_gen_bswap16_i64(swap
, val
, 0);
3025 tcg_gen_bswap32_i64(swap
, val
, 0);
3028 tcg_gen_bswap64_i64(swap
, val
);
3031 g_assert_not_reached();
3037 addr
= plugin_prep_mem_callbacks(addr
);
3038 gen_ldst_i64(INDEX_op_qemu_st_i64
, val
, addr
, memop
, idx
);
3039 plugin_gen_mem_callbacks(addr
, oi
, QEMU_PLUGIN_MEM_W
);
3042 tcg_temp_free_i64(swap
);
3046 static void tcg_gen_ext_i32(TCGv_i32 ret
, TCGv_i32 val
, MemOp opc
)
3048 switch (opc
& MO_SSIZE
) {
3050 tcg_gen_ext8s_i32(ret
, val
);
3053 tcg_gen_ext8u_i32(ret
, val
);
3056 tcg_gen_ext16s_i32(ret
, val
);
3059 tcg_gen_ext16u_i32(ret
, val
);
3062 tcg_gen_mov_i32(ret
, val
);
3067 static void tcg_gen_ext_i64(TCGv_i64 ret
, TCGv_i64 val
, MemOp opc
)
3069 switch (opc
& MO_SSIZE
) {
3071 tcg_gen_ext8s_i64(ret
, val
);
3074 tcg_gen_ext8u_i64(ret
, val
);
3077 tcg_gen_ext16s_i64(ret
, val
);
3080 tcg_gen_ext16u_i64(ret
, val
);
3083 tcg_gen_ext32s_i64(ret
, val
);
3086 tcg_gen_ext32u_i64(ret
, val
);
3089 tcg_gen_mov_i64(ret
, val
);
3094 typedef void (*gen_atomic_cx_i32
)(TCGv_i32
, TCGv_env
, TCGv
,
3095 TCGv_i32
, TCGv_i32
, TCGv_i32
);
3096 typedef void (*gen_atomic_cx_i64
)(TCGv_i64
, TCGv_env
, TCGv
,
3097 TCGv_i64
, TCGv_i64
, TCGv_i32
);
3098 typedef void (*gen_atomic_op_i32
)(TCGv_i32
, TCGv_env
, TCGv
,
3099 TCGv_i32
, TCGv_i32
);
3100 typedef void (*gen_atomic_op_i64
)(TCGv_i64
, TCGv_env
, TCGv
,
3101 TCGv_i64
, TCGv_i32
);
3103 #ifdef CONFIG_ATOMIC64
3104 # define WITH_ATOMIC64(X) X,
3106 # define WITH_ATOMIC64(X)
3109 static void * const table_cmpxchg
[(MO_SIZE
| MO_BSWAP
) + 1] = {
3110 [MO_8
] = gen_helper_atomic_cmpxchgb
,
3111 [MO_16
| MO_LE
] = gen_helper_atomic_cmpxchgw_le
,
3112 [MO_16
| MO_BE
] = gen_helper_atomic_cmpxchgw_be
,
3113 [MO_32
| MO_LE
] = gen_helper_atomic_cmpxchgl_le
,
3114 [MO_32
| MO_BE
] = gen_helper_atomic_cmpxchgl_be
,
3115 WITH_ATOMIC64([MO_64
| MO_LE
] = gen_helper_atomic_cmpxchgq_le
)
3116 WITH_ATOMIC64([MO_64
| MO_BE
] = gen_helper_atomic_cmpxchgq_be
)
3119 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv
, TCGv addr
, TCGv_i32 cmpv
,
3120 TCGv_i32 newv
, TCGArg idx
, MemOp memop
)
3122 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3124 if (!(tcg_ctx
->tb_cflags
& CF_PARALLEL
)) {
3125 TCGv_i32 t1
= tcg_temp_new_i32();
3126 TCGv_i32 t2
= tcg_temp_new_i32();
3128 tcg_gen_ext_i32(t2
, cmpv
, memop
& MO_SIZE
);
3130 tcg_gen_qemu_ld_i32(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3131 tcg_gen_movcond_i32(TCG_COND_EQ
, t2
, t1
, t2
, newv
, t1
);
3132 tcg_gen_qemu_st_i32(t2
, addr
, idx
, memop
);
3133 tcg_temp_free_i32(t2
);
3135 if (memop
& MO_SIGN
) {
3136 tcg_gen_ext_i32(retv
, t1
, memop
);
3138 tcg_gen_mov_i32(retv
, t1
);
3140 tcg_temp_free_i32(t1
);
3142 gen_atomic_cx_i32 gen
;
3145 gen
= table_cmpxchg
[memop
& (MO_SIZE
| MO_BSWAP
)];
3146 tcg_debug_assert(gen
!= NULL
);
3148 oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3149 gen(retv
, cpu_env
, addr
, cmpv
, newv
, tcg_constant_i32(oi
));
3151 if (memop
& MO_SIGN
) {
3152 tcg_gen_ext_i32(retv
, retv
, memop
);
3157 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv
, TCGv addr
, TCGv_i64 cmpv
,
3158 TCGv_i64 newv
, TCGArg idx
, MemOp memop
)
3160 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3162 if (!(tcg_ctx
->tb_cflags
& CF_PARALLEL
)) {
3163 TCGv_i64 t1
= tcg_temp_new_i64();
3164 TCGv_i64 t2
= tcg_temp_new_i64();
3166 tcg_gen_ext_i64(t2
, cmpv
, memop
& MO_SIZE
);
3168 tcg_gen_qemu_ld_i64(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3169 tcg_gen_movcond_i64(TCG_COND_EQ
, t2
, t1
, t2
, newv
, t1
);
3170 tcg_gen_qemu_st_i64(t2
, addr
, idx
, memop
);
3171 tcg_temp_free_i64(t2
);
3173 if (memop
& MO_SIGN
) {
3174 tcg_gen_ext_i64(retv
, t1
, memop
);
3176 tcg_gen_mov_i64(retv
, t1
);
3178 tcg_temp_free_i64(t1
);
3179 } else if ((memop
& MO_SIZE
) == MO_64
) {
3180 #ifdef CONFIG_ATOMIC64
3181 gen_atomic_cx_i64 gen
;
3184 gen
= table_cmpxchg
[memop
& (MO_SIZE
| MO_BSWAP
)];
3185 tcg_debug_assert(gen
!= NULL
);
3187 oi
= make_memop_idx(memop
, idx
);
3188 gen(retv
, cpu_env
, addr
, cmpv
, newv
, tcg_constant_i32(oi
));
3190 gen_helper_exit_atomic(cpu_env
);
3191 /* Produce a result, so that we have a well-formed opcode stream
3192 with respect to uses of the result in the (dead) code following. */
3193 tcg_gen_movi_i64(retv
, 0);
3194 #endif /* CONFIG_ATOMIC64 */
3196 TCGv_i32 c32
= tcg_temp_new_i32();
3197 TCGv_i32 n32
= tcg_temp_new_i32();
3198 TCGv_i32 r32
= tcg_temp_new_i32();
3200 tcg_gen_extrl_i64_i32(c32
, cmpv
);
3201 tcg_gen_extrl_i64_i32(n32
, newv
);
3202 tcg_gen_atomic_cmpxchg_i32(r32
, addr
, c32
, n32
, idx
, memop
& ~MO_SIGN
);
3203 tcg_temp_free_i32(c32
);
3204 tcg_temp_free_i32(n32
);
3206 tcg_gen_extu_i32_i64(retv
, r32
);
3207 tcg_temp_free_i32(r32
);
3209 if (memop
& MO_SIGN
) {
3210 tcg_gen_ext_i64(retv
, retv
, memop
);
3215 static void do_nonatomic_op_i32(TCGv_i32 ret
, TCGv addr
, TCGv_i32 val
,
3216 TCGArg idx
, MemOp memop
, bool new_val
,
3217 void (*gen
)(TCGv_i32
, TCGv_i32
, TCGv_i32
))
3219 TCGv_i32 t1
= tcg_temp_new_i32();
3220 TCGv_i32 t2
= tcg_temp_new_i32();
3222 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3224 tcg_gen_qemu_ld_i32(t1
, addr
, idx
, memop
);
3225 tcg_gen_ext_i32(t2
, val
, memop
);
3227 tcg_gen_qemu_st_i32(t2
, addr
, idx
, memop
);
3229 tcg_gen_ext_i32(ret
, (new_val
? t2
: t1
), memop
);
3230 tcg_temp_free_i32(t1
);
3231 tcg_temp_free_i32(t2
);
3234 static void do_atomic_op_i32(TCGv_i32 ret
, TCGv addr
, TCGv_i32 val
,
3235 TCGArg idx
, MemOp memop
, void * const table
[])
3237 gen_atomic_op_i32 gen
;
3240 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3242 gen
= table
[memop
& (MO_SIZE
| MO_BSWAP
)];
3243 tcg_debug_assert(gen
!= NULL
);
3245 oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3246 gen(ret
, cpu_env
, addr
, val
, tcg_constant_i32(oi
));
3248 if (memop
& MO_SIGN
) {
3249 tcg_gen_ext_i32(ret
, ret
, memop
);
3253 static void do_nonatomic_op_i64(TCGv_i64 ret
, TCGv addr
, TCGv_i64 val
,
3254 TCGArg idx
, MemOp memop
, bool new_val
,
3255 void (*gen
)(TCGv_i64
, TCGv_i64
, TCGv_i64
))
3257 TCGv_i64 t1
= tcg_temp_new_i64();
3258 TCGv_i64 t2
= tcg_temp_new_i64();
3260 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3262 tcg_gen_qemu_ld_i64(t1
, addr
, idx
, memop
);
3263 tcg_gen_ext_i64(t2
, val
, memop
);
3265 tcg_gen_qemu_st_i64(t2
, addr
, idx
, memop
);
3267 tcg_gen_ext_i64(ret
, (new_val
? t2
: t1
), memop
);
3268 tcg_temp_free_i64(t1
);
3269 tcg_temp_free_i64(t2
);
3272 static void do_atomic_op_i64(TCGv_i64 ret
, TCGv addr
, TCGv_i64 val
,
3273 TCGArg idx
, MemOp memop
, void * const table
[])
3275 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3277 if ((memop
& MO_SIZE
) == MO_64
) {
3278 #ifdef CONFIG_ATOMIC64
3279 gen_atomic_op_i64 gen
;
3282 gen
= table
[memop
& (MO_SIZE
| MO_BSWAP
)];
3283 tcg_debug_assert(gen
!= NULL
);
3285 oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3286 gen(ret
, cpu_env
, addr
, val
, tcg_constant_i32(oi
));
3288 gen_helper_exit_atomic(cpu_env
);
3289 /* Produce a result, so that we have a well-formed opcode stream
3290 with respect to uses of the result in the (dead) code following. */
3291 tcg_gen_movi_i64(ret
, 0);
3292 #endif /* CONFIG_ATOMIC64 */
3294 TCGv_i32 v32
= tcg_temp_new_i32();
3295 TCGv_i32 r32
= tcg_temp_new_i32();
3297 tcg_gen_extrl_i64_i32(v32
, val
);
3298 do_atomic_op_i32(r32
, addr
, v32
, idx
, memop
& ~MO_SIGN
, table
);
3299 tcg_temp_free_i32(v32
);
3301 tcg_gen_extu_i32_i64(ret
, r32
);
3302 tcg_temp_free_i32(r32
);
3304 if (memop
& MO_SIGN
) {
3305 tcg_gen_ext_i64(ret
, ret
, memop
);
3310 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3311 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
3312 [MO_8] = gen_helper_atomic_##NAME##b, \
3313 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3314 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3315 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3316 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
3317 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3318 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
3320 void tcg_gen_atomic_##NAME##_i32 \
3321 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \
3323 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3324 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3326 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3327 tcg_gen_##OP##_i32); \
3330 void tcg_gen_atomic_##NAME##_i64 \
3331 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \
3333 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3334 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3336 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3337 tcg_gen_##OP##_i64); \
3341 GEN_ATOMIC_HELPER(fetch_add
, add
, 0)
3342 GEN_ATOMIC_HELPER(fetch_and
, and, 0)
3343 GEN_ATOMIC_HELPER(fetch_or
, or, 0)
3344 GEN_ATOMIC_HELPER(fetch_xor
, xor, 0)
3345 GEN_ATOMIC_HELPER(fetch_smin
, smin
, 0)
3346 GEN_ATOMIC_HELPER(fetch_umin
, umin
, 0)
3347 GEN_ATOMIC_HELPER(fetch_smax
, smax
, 0)
3348 GEN_ATOMIC_HELPER(fetch_umax
, umax
, 0)
3350 GEN_ATOMIC_HELPER(add_fetch
, add
, 1)
3351 GEN_ATOMIC_HELPER(and_fetch
, and, 1)
3352 GEN_ATOMIC_HELPER(or_fetch
, or, 1)
3353 GEN_ATOMIC_HELPER(xor_fetch
, xor, 1)
3354 GEN_ATOMIC_HELPER(smin_fetch
, smin
, 1)
3355 GEN_ATOMIC_HELPER(umin_fetch
, umin
, 1)
3356 GEN_ATOMIC_HELPER(smax_fetch
, smax
, 1)
3357 GEN_ATOMIC_HELPER(umax_fetch
, umax
, 1)
3359 static void tcg_gen_mov2_i32(TCGv_i32 r
, TCGv_i32 a
, TCGv_i32 b
)
3361 tcg_gen_mov_i32(r
, b
);
3364 static void tcg_gen_mov2_i64(TCGv_i64 r
, TCGv_i64 a
, TCGv_i64 b
)
3366 tcg_gen_mov_i64(r
, b
);
3369 GEN_ATOMIC_HELPER(xchg
, mov2
, 0)
3371 #undef GEN_ATOMIC_HELPER