2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2008 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 #include "qemu/osdep.h"
26 #include "exec/exec-all.h"
28 #include "tcg/tcg-op.h"
29 #include "tcg/tcg-mo.h"
30 #include "exec/plugin-gen.h"
31 #include "tcg-internal.h"
34 void tcg_gen_op1(TCGOpcode opc
, TCGArg a1
)
36 TCGOp
*op
= tcg_emit_op(opc
, 1);
40 void tcg_gen_op2(TCGOpcode opc
, TCGArg a1
, TCGArg a2
)
42 TCGOp
*op
= tcg_emit_op(opc
, 2);
47 void tcg_gen_op3(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
)
49 TCGOp
*op
= tcg_emit_op(opc
, 3);
55 void tcg_gen_op4(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
, TCGArg a4
)
57 TCGOp
*op
= tcg_emit_op(opc
, 4);
64 void tcg_gen_op5(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
,
67 TCGOp
*op
= tcg_emit_op(opc
, 5);
75 void tcg_gen_op6(TCGOpcode opc
, TCGArg a1
, TCGArg a2
, TCGArg a3
,
76 TCGArg a4
, TCGArg a5
, TCGArg a6
)
78 TCGOp
*op
= tcg_emit_op(opc
, 6);
87 void tcg_gen_mb(TCGBar mb_type
)
89 if (tcg_ctx
->tb_cflags
& CF_PARALLEL
) {
90 tcg_gen_op1(INDEX_op_mb
, mb_type
);
96 void tcg_gen_movi_i32(TCGv_i32 ret
, int32_t arg
)
98 tcg_gen_mov_i32(ret
, tcg_constant_i32(arg
));
101 void tcg_gen_addi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
103 /* some cases can be optimized here */
105 tcg_gen_mov_i32(ret
, arg1
);
107 tcg_gen_add_i32(ret
, arg1
, tcg_constant_i32(arg2
));
111 void tcg_gen_subfi_i32(TCGv_i32 ret
, int32_t arg1
, TCGv_i32 arg2
)
113 if (arg1
== 0 && TCG_TARGET_HAS_neg_i32
) {
114 /* Don't recurse with tcg_gen_neg_i32. */
115 tcg_gen_op2_i32(INDEX_op_neg_i32
, ret
, arg2
);
117 tcg_gen_sub_i32(ret
, tcg_constant_i32(arg1
), arg2
);
121 void tcg_gen_subi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
123 /* some cases can be optimized here */
125 tcg_gen_mov_i32(ret
, arg1
);
127 tcg_gen_sub_i32(ret
, arg1
, tcg_constant_i32(arg2
));
131 void tcg_gen_andi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
133 /* Some cases can be optimized here. */
136 tcg_gen_movi_i32(ret
, 0);
139 tcg_gen_mov_i32(ret
, arg1
);
142 /* Don't recurse with tcg_gen_ext8u_i32. */
143 if (TCG_TARGET_HAS_ext8u_i32
) {
144 tcg_gen_op2_i32(INDEX_op_ext8u_i32
, ret
, arg1
);
149 if (TCG_TARGET_HAS_ext16u_i32
) {
150 tcg_gen_op2_i32(INDEX_op_ext16u_i32
, ret
, arg1
);
156 tcg_gen_and_i32(ret
, arg1
, tcg_constant_i32(arg2
));
159 void tcg_gen_ori_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
161 /* Some cases can be optimized here. */
163 tcg_gen_movi_i32(ret
, -1);
164 } else if (arg2
== 0) {
165 tcg_gen_mov_i32(ret
, arg1
);
167 tcg_gen_or_i32(ret
, arg1
, tcg_constant_i32(arg2
));
171 void tcg_gen_xori_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
173 /* Some cases can be optimized here. */
175 tcg_gen_mov_i32(ret
, arg1
);
176 } else if (arg2
== -1 && TCG_TARGET_HAS_not_i32
) {
177 /* Don't recurse with tcg_gen_not_i32. */
178 tcg_gen_op2_i32(INDEX_op_not_i32
, ret
, arg1
);
180 tcg_gen_xor_i32(ret
, arg1
, tcg_constant_i32(arg2
));
184 void tcg_gen_shli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
186 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
188 tcg_gen_mov_i32(ret
, arg1
);
190 tcg_gen_shl_i32(ret
, arg1
, tcg_constant_i32(arg2
));
194 void tcg_gen_shri_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
196 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
198 tcg_gen_mov_i32(ret
, arg1
);
200 tcg_gen_shr_i32(ret
, arg1
, tcg_constant_i32(arg2
));
204 void tcg_gen_sari_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
206 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
208 tcg_gen_mov_i32(ret
, arg1
);
210 tcg_gen_sar_i32(ret
, arg1
, tcg_constant_i32(arg2
));
214 void tcg_gen_brcond_i32(TCGCond cond
, TCGv_i32 arg1
, TCGv_i32 arg2
, TCGLabel
*l
)
216 if (cond
== TCG_COND_ALWAYS
) {
218 } else if (cond
!= TCG_COND_NEVER
) {
220 tcg_gen_op4ii_i32(INDEX_op_brcond_i32
, arg1
, arg2
, cond
, label_arg(l
));
224 void tcg_gen_brcondi_i32(TCGCond cond
, TCGv_i32 arg1
, int32_t arg2
, TCGLabel
*l
)
226 if (cond
== TCG_COND_ALWAYS
) {
228 } else if (cond
!= TCG_COND_NEVER
) {
229 tcg_gen_brcond_i32(cond
, arg1
, tcg_constant_i32(arg2
), l
);
233 void tcg_gen_setcond_i32(TCGCond cond
, TCGv_i32 ret
,
234 TCGv_i32 arg1
, TCGv_i32 arg2
)
236 if (cond
== TCG_COND_ALWAYS
) {
237 tcg_gen_movi_i32(ret
, 1);
238 } else if (cond
== TCG_COND_NEVER
) {
239 tcg_gen_movi_i32(ret
, 0);
241 tcg_gen_op4i_i32(INDEX_op_setcond_i32
, ret
, arg1
, arg2
, cond
);
245 void tcg_gen_setcondi_i32(TCGCond cond
, TCGv_i32 ret
,
246 TCGv_i32 arg1
, int32_t arg2
)
248 tcg_gen_setcond_i32(cond
, ret
, arg1
, tcg_constant_i32(arg2
));
251 void tcg_gen_muli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
254 tcg_gen_movi_i32(ret
, 0);
255 } else if (is_power_of_2(arg2
)) {
256 tcg_gen_shli_i32(ret
, arg1
, ctz32(arg2
));
258 tcg_gen_mul_i32(ret
, arg1
, tcg_constant_i32(arg2
));
262 void tcg_gen_div_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
264 if (TCG_TARGET_HAS_div_i32
) {
265 tcg_gen_op3_i32(INDEX_op_div_i32
, ret
, arg1
, arg2
);
266 } else if (TCG_TARGET_HAS_div2_i32
) {
267 TCGv_i32 t0
= tcg_temp_new_i32();
268 tcg_gen_sari_i32(t0
, arg1
, 31);
269 tcg_gen_op5_i32(INDEX_op_div2_i32
, ret
, t0
, arg1
, t0
, arg2
);
270 tcg_temp_free_i32(t0
);
272 gen_helper_div_i32(ret
, arg1
, arg2
);
276 void tcg_gen_rem_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
278 if (TCG_TARGET_HAS_rem_i32
) {
279 tcg_gen_op3_i32(INDEX_op_rem_i32
, ret
, arg1
, arg2
);
280 } else if (TCG_TARGET_HAS_div_i32
) {
281 TCGv_i32 t0
= tcg_temp_new_i32();
282 tcg_gen_op3_i32(INDEX_op_div_i32
, t0
, arg1
, arg2
);
283 tcg_gen_mul_i32(t0
, t0
, arg2
);
284 tcg_gen_sub_i32(ret
, arg1
, t0
);
285 tcg_temp_free_i32(t0
);
286 } else if (TCG_TARGET_HAS_div2_i32
) {
287 TCGv_i32 t0
= tcg_temp_new_i32();
288 tcg_gen_sari_i32(t0
, arg1
, 31);
289 tcg_gen_op5_i32(INDEX_op_div2_i32
, t0
, ret
, arg1
, t0
, arg2
);
290 tcg_temp_free_i32(t0
);
292 gen_helper_rem_i32(ret
, arg1
, arg2
);
296 void tcg_gen_divu_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
298 if (TCG_TARGET_HAS_div_i32
) {
299 tcg_gen_op3_i32(INDEX_op_divu_i32
, ret
, arg1
, arg2
);
300 } else if (TCG_TARGET_HAS_div2_i32
) {
301 TCGv_i32 t0
= tcg_temp_new_i32();
302 tcg_gen_movi_i32(t0
, 0);
303 tcg_gen_op5_i32(INDEX_op_divu2_i32
, ret
, t0
, arg1
, t0
, arg2
);
304 tcg_temp_free_i32(t0
);
306 gen_helper_divu_i32(ret
, arg1
, arg2
);
310 void tcg_gen_remu_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
312 if (TCG_TARGET_HAS_rem_i32
) {
313 tcg_gen_op3_i32(INDEX_op_remu_i32
, ret
, arg1
, arg2
);
314 } else if (TCG_TARGET_HAS_div_i32
) {
315 TCGv_i32 t0
= tcg_temp_new_i32();
316 tcg_gen_op3_i32(INDEX_op_divu_i32
, t0
, arg1
, arg2
);
317 tcg_gen_mul_i32(t0
, t0
, arg2
);
318 tcg_gen_sub_i32(ret
, arg1
, t0
);
319 tcg_temp_free_i32(t0
);
320 } else if (TCG_TARGET_HAS_div2_i32
) {
321 TCGv_i32 t0
= tcg_temp_new_i32();
322 tcg_gen_movi_i32(t0
, 0);
323 tcg_gen_op5_i32(INDEX_op_divu2_i32
, t0
, ret
, arg1
, t0
, arg2
);
324 tcg_temp_free_i32(t0
);
326 gen_helper_remu_i32(ret
, arg1
, arg2
);
330 void tcg_gen_andc_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
332 if (TCG_TARGET_HAS_andc_i32
) {
333 tcg_gen_op3_i32(INDEX_op_andc_i32
, ret
, arg1
, arg2
);
335 TCGv_i32 t0
= tcg_temp_new_i32();
336 tcg_gen_not_i32(t0
, arg2
);
337 tcg_gen_and_i32(ret
, arg1
, t0
);
338 tcg_temp_free_i32(t0
);
342 void tcg_gen_eqv_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
344 if (TCG_TARGET_HAS_eqv_i32
) {
345 tcg_gen_op3_i32(INDEX_op_eqv_i32
, ret
, arg1
, arg2
);
347 tcg_gen_xor_i32(ret
, arg1
, arg2
);
348 tcg_gen_not_i32(ret
, ret
);
352 void tcg_gen_nand_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
354 if (TCG_TARGET_HAS_nand_i32
) {
355 tcg_gen_op3_i32(INDEX_op_nand_i32
, ret
, arg1
, arg2
);
357 tcg_gen_and_i32(ret
, arg1
, arg2
);
358 tcg_gen_not_i32(ret
, ret
);
362 void tcg_gen_nor_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
364 if (TCG_TARGET_HAS_nor_i32
) {
365 tcg_gen_op3_i32(INDEX_op_nor_i32
, ret
, arg1
, arg2
);
367 tcg_gen_or_i32(ret
, arg1
, arg2
);
368 tcg_gen_not_i32(ret
, ret
);
372 void tcg_gen_orc_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
374 if (TCG_TARGET_HAS_orc_i32
) {
375 tcg_gen_op3_i32(INDEX_op_orc_i32
, ret
, arg1
, arg2
);
377 TCGv_i32 t0
= tcg_temp_new_i32();
378 tcg_gen_not_i32(t0
, arg2
);
379 tcg_gen_or_i32(ret
, arg1
, t0
);
380 tcg_temp_free_i32(t0
);
384 void tcg_gen_clz_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
386 if (TCG_TARGET_HAS_clz_i32
) {
387 tcg_gen_op3_i32(INDEX_op_clz_i32
, ret
, arg1
, arg2
);
388 } else if (TCG_TARGET_HAS_clz_i64
) {
389 TCGv_i64 t1
= tcg_temp_new_i64();
390 TCGv_i64 t2
= tcg_temp_new_i64();
391 tcg_gen_extu_i32_i64(t1
, arg1
);
392 tcg_gen_extu_i32_i64(t2
, arg2
);
393 tcg_gen_addi_i64(t2
, t2
, 32);
394 tcg_gen_clz_i64(t1
, t1
, t2
);
395 tcg_gen_extrl_i64_i32(ret
, t1
);
396 tcg_temp_free_i64(t1
);
397 tcg_temp_free_i64(t2
);
398 tcg_gen_subi_i32(ret
, ret
, 32);
400 gen_helper_clz_i32(ret
, arg1
, arg2
);
404 void tcg_gen_clzi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, uint32_t arg2
)
406 tcg_gen_clz_i32(ret
, arg1
, tcg_constant_i32(arg2
));
409 void tcg_gen_ctz_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
411 if (TCG_TARGET_HAS_ctz_i32
) {
412 tcg_gen_op3_i32(INDEX_op_ctz_i32
, ret
, arg1
, arg2
);
413 } else if (TCG_TARGET_HAS_ctz_i64
) {
414 TCGv_i64 t1
= tcg_temp_new_i64();
415 TCGv_i64 t2
= tcg_temp_new_i64();
416 tcg_gen_extu_i32_i64(t1
, arg1
);
417 tcg_gen_extu_i32_i64(t2
, arg2
);
418 tcg_gen_ctz_i64(t1
, t1
, t2
);
419 tcg_gen_extrl_i64_i32(ret
, t1
);
420 tcg_temp_free_i64(t1
);
421 tcg_temp_free_i64(t2
);
422 } else if (TCG_TARGET_HAS_ctpop_i32
423 || TCG_TARGET_HAS_ctpop_i64
424 || TCG_TARGET_HAS_clz_i32
425 || TCG_TARGET_HAS_clz_i64
) {
426 TCGv_i32 z
, t
= tcg_temp_new_i32();
428 if (TCG_TARGET_HAS_ctpop_i32
|| TCG_TARGET_HAS_ctpop_i64
) {
429 tcg_gen_subi_i32(t
, arg1
, 1);
430 tcg_gen_andc_i32(t
, t
, arg1
);
431 tcg_gen_ctpop_i32(t
, t
);
433 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
434 tcg_gen_neg_i32(t
, arg1
);
435 tcg_gen_and_i32(t
, t
, arg1
);
436 tcg_gen_clzi_i32(t
, t
, 32);
437 tcg_gen_xori_i32(t
, t
, 31);
439 z
= tcg_constant_i32(0);
440 tcg_gen_movcond_i32(TCG_COND_EQ
, ret
, arg1
, z
, arg2
, t
);
441 tcg_temp_free_i32(t
);
443 gen_helper_ctz_i32(ret
, arg1
, arg2
);
447 void tcg_gen_ctzi_i32(TCGv_i32 ret
, TCGv_i32 arg1
, uint32_t arg2
)
449 if (!TCG_TARGET_HAS_ctz_i32
&& TCG_TARGET_HAS_ctpop_i32
&& arg2
== 32) {
450 /* This equivalence has the advantage of not requiring a fixup. */
451 TCGv_i32 t
= tcg_temp_new_i32();
452 tcg_gen_subi_i32(t
, arg1
, 1);
453 tcg_gen_andc_i32(t
, t
, arg1
);
454 tcg_gen_ctpop_i32(ret
, t
);
455 tcg_temp_free_i32(t
);
457 tcg_gen_ctz_i32(ret
, arg1
, tcg_constant_i32(arg2
));
461 void tcg_gen_clrsb_i32(TCGv_i32 ret
, TCGv_i32 arg
)
463 if (TCG_TARGET_HAS_clz_i32
) {
464 TCGv_i32 t
= tcg_temp_new_i32();
465 tcg_gen_sari_i32(t
, arg
, 31);
466 tcg_gen_xor_i32(t
, t
, arg
);
467 tcg_gen_clzi_i32(t
, t
, 32);
468 tcg_gen_subi_i32(ret
, t
, 1);
469 tcg_temp_free_i32(t
);
471 gen_helper_clrsb_i32(ret
, arg
);
475 void tcg_gen_ctpop_i32(TCGv_i32 ret
, TCGv_i32 arg1
)
477 if (TCG_TARGET_HAS_ctpop_i32
) {
478 tcg_gen_op2_i32(INDEX_op_ctpop_i32
, ret
, arg1
);
479 } else if (TCG_TARGET_HAS_ctpop_i64
) {
480 TCGv_i64 t
= tcg_temp_new_i64();
481 tcg_gen_extu_i32_i64(t
, arg1
);
482 tcg_gen_ctpop_i64(t
, t
);
483 tcg_gen_extrl_i64_i32(ret
, t
);
484 tcg_temp_free_i64(t
);
486 gen_helper_ctpop_i32(ret
, arg1
);
490 void tcg_gen_rotl_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
492 if (TCG_TARGET_HAS_rot_i32
) {
493 tcg_gen_op3_i32(INDEX_op_rotl_i32
, ret
, arg1
, arg2
);
497 t0
= tcg_temp_new_i32();
498 t1
= tcg_temp_new_i32();
499 tcg_gen_shl_i32(t0
, arg1
, arg2
);
500 tcg_gen_subfi_i32(t1
, 32, arg2
);
501 tcg_gen_shr_i32(t1
, arg1
, t1
);
502 tcg_gen_or_i32(ret
, t0
, t1
);
503 tcg_temp_free_i32(t0
);
504 tcg_temp_free_i32(t1
);
508 void tcg_gen_rotli_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
510 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
511 /* some cases can be optimized here */
513 tcg_gen_mov_i32(ret
, arg1
);
514 } else if (TCG_TARGET_HAS_rot_i32
) {
515 tcg_gen_rotl_i32(ret
, arg1
, tcg_constant_i32(arg2
));
518 t0
= tcg_temp_new_i32();
519 t1
= tcg_temp_new_i32();
520 tcg_gen_shli_i32(t0
, arg1
, arg2
);
521 tcg_gen_shri_i32(t1
, arg1
, 32 - arg2
);
522 tcg_gen_or_i32(ret
, t0
, t1
);
523 tcg_temp_free_i32(t0
);
524 tcg_temp_free_i32(t1
);
528 void tcg_gen_rotr_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
)
530 if (TCG_TARGET_HAS_rot_i32
) {
531 tcg_gen_op3_i32(INDEX_op_rotr_i32
, ret
, arg1
, arg2
);
535 t0
= tcg_temp_new_i32();
536 t1
= tcg_temp_new_i32();
537 tcg_gen_shr_i32(t0
, arg1
, arg2
);
538 tcg_gen_subfi_i32(t1
, 32, arg2
);
539 tcg_gen_shl_i32(t1
, arg1
, t1
);
540 tcg_gen_or_i32(ret
, t0
, t1
);
541 tcg_temp_free_i32(t0
);
542 tcg_temp_free_i32(t1
);
546 void tcg_gen_rotri_i32(TCGv_i32 ret
, TCGv_i32 arg1
, int32_t arg2
)
548 tcg_debug_assert(arg2
>= 0 && arg2
< 32);
549 /* some cases can be optimized here */
551 tcg_gen_mov_i32(ret
, arg1
);
553 tcg_gen_rotli_i32(ret
, arg1
, 32 - arg2
);
557 void tcg_gen_deposit_i32(TCGv_i32 ret
, TCGv_i32 arg1
, TCGv_i32 arg2
,
558 unsigned int ofs
, unsigned int len
)
563 tcg_debug_assert(ofs
< 32);
564 tcg_debug_assert(len
> 0);
565 tcg_debug_assert(len
<= 32);
566 tcg_debug_assert(ofs
+ len
<= 32);
569 tcg_gen_mov_i32(ret
, arg2
);
572 if (TCG_TARGET_HAS_deposit_i32
&& TCG_TARGET_deposit_i32_valid(ofs
, len
)) {
573 tcg_gen_op5ii_i32(INDEX_op_deposit_i32
, ret
, arg1
, arg2
, ofs
, len
);
577 t1
= tcg_temp_new_i32();
579 if (TCG_TARGET_HAS_extract2_i32
) {
580 if (ofs
+ len
== 32) {
581 tcg_gen_shli_i32(t1
, arg1
, len
);
582 tcg_gen_extract2_i32(ret
, t1
, arg2
, len
);
586 tcg_gen_extract2_i32(ret
, arg1
, arg2
, len
);
587 tcg_gen_rotli_i32(ret
, ret
, len
);
592 mask
= (1u << len
) - 1;
593 if (ofs
+ len
< 32) {
594 tcg_gen_andi_i32(t1
, arg2
, mask
);
595 tcg_gen_shli_i32(t1
, t1
, ofs
);
597 tcg_gen_shli_i32(t1
, arg2
, ofs
);
599 tcg_gen_andi_i32(ret
, arg1
, ~(mask
<< ofs
));
600 tcg_gen_or_i32(ret
, ret
, t1
);
602 tcg_temp_free_i32(t1
);
605 void tcg_gen_deposit_z_i32(TCGv_i32 ret
, TCGv_i32 arg
,
606 unsigned int ofs
, unsigned int len
)
608 tcg_debug_assert(ofs
< 32);
609 tcg_debug_assert(len
> 0);
610 tcg_debug_assert(len
<= 32);
611 tcg_debug_assert(ofs
+ len
<= 32);
613 if (ofs
+ len
== 32) {
614 tcg_gen_shli_i32(ret
, arg
, ofs
);
615 } else if (ofs
== 0) {
616 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
617 } else if (TCG_TARGET_HAS_deposit_i32
618 && TCG_TARGET_deposit_i32_valid(ofs
, len
)) {
619 TCGv_i32 zero
= tcg_constant_i32(0);
620 tcg_gen_op5ii_i32(INDEX_op_deposit_i32
, ret
, zero
, arg
, ofs
, len
);
622 /* To help two-operand hosts we prefer to zero-extend first,
623 which allows ARG to stay live. */
626 if (TCG_TARGET_HAS_ext16u_i32
) {
627 tcg_gen_ext16u_i32(ret
, arg
);
628 tcg_gen_shli_i32(ret
, ret
, ofs
);
633 if (TCG_TARGET_HAS_ext8u_i32
) {
634 tcg_gen_ext8u_i32(ret
, arg
);
635 tcg_gen_shli_i32(ret
, ret
, ofs
);
640 /* Otherwise prefer zero-extension over AND for code size. */
643 if (TCG_TARGET_HAS_ext16u_i32
) {
644 tcg_gen_shli_i32(ret
, arg
, ofs
);
645 tcg_gen_ext16u_i32(ret
, ret
);
650 if (TCG_TARGET_HAS_ext8u_i32
) {
651 tcg_gen_shli_i32(ret
, arg
, ofs
);
652 tcg_gen_ext8u_i32(ret
, ret
);
657 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
658 tcg_gen_shli_i32(ret
, ret
, ofs
);
662 void tcg_gen_extract_i32(TCGv_i32 ret
, TCGv_i32 arg
,
663 unsigned int ofs
, unsigned int len
)
665 tcg_debug_assert(ofs
< 32);
666 tcg_debug_assert(len
> 0);
667 tcg_debug_assert(len
<= 32);
668 tcg_debug_assert(ofs
+ len
<= 32);
670 /* Canonicalize certain special cases, even if extract is supported. */
671 if (ofs
+ len
== 32) {
672 tcg_gen_shri_i32(ret
, arg
, 32 - len
);
676 tcg_gen_andi_i32(ret
, arg
, (1u << len
) - 1);
680 if (TCG_TARGET_HAS_extract_i32
681 && TCG_TARGET_extract_i32_valid(ofs
, len
)) {
682 tcg_gen_op4ii_i32(INDEX_op_extract_i32
, ret
, arg
, ofs
, len
);
686 /* Assume that zero-extension, if available, is cheaper than a shift. */
689 if (TCG_TARGET_HAS_ext16u_i32
) {
690 tcg_gen_ext16u_i32(ret
, arg
);
691 tcg_gen_shri_i32(ret
, ret
, ofs
);
696 if (TCG_TARGET_HAS_ext8u_i32
) {
697 tcg_gen_ext8u_i32(ret
, arg
);
698 tcg_gen_shri_i32(ret
, ret
, ofs
);
704 /* ??? Ideally we'd know what values are available for immediate AND.
705 Assume that 8 bits are available, plus the special case of 16,
706 so that we get ext8u, ext16u. */
708 case 1 ... 8: case 16:
709 tcg_gen_shri_i32(ret
, arg
, ofs
);
710 tcg_gen_andi_i32(ret
, ret
, (1u << len
) - 1);
713 tcg_gen_shli_i32(ret
, arg
, 32 - len
- ofs
);
714 tcg_gen_shri_i32(ret
, ret
, 32 - len
);
719 void tcg_gen_sextract_i32(TCGv_i32 ret
, TCGv_i32 arg
,
720 unsigned int ofs
, unsigned int len
)
722 tcg_debug_assert(ofs
< 32);
723 tcg_debug_assert(len
> 0);
724 tcg_debug_assert(len
<= 32);
725 tcg_debug_assert(ofs
+ len
<= 32);
727 /* Canonicalize certain special cases, even if extract is supported. */
728 if (ofs
+ len
== 32) {
729 tcg_gen_sari_i32(ret
, arg
, 32 - len
);
735 tcg_gen_ext16s_i32(ret
, arg
);
738 tcg_gen_ext8s_i32(ret
, arg
);
743 if (TCG_TARGET_HAS_sextract_i32
744 && TCG_TARGET_extract_i32_valid(ofs
, len
)) {
745 tcg_gen_op4ii_i32(INDEX_op_sextract_i32
, ret
, arg
, ofs
, len
);
749 /* Assume that sign-extension, if available, is cheaper than a shift. */
752 if (TCG_TARGET_HAS_ext16s_i32
) {
753 tcg_gen_ext16s_i32(ret
, arg
);
754 tcg_gen_sari_i32(ret
, ret
, ofs
);
759 if (TCG_TARGET_HAS_ext8s_i32
) {
760 tcg_gen_ext8s_i32(ret
, arg
);
761 tcg_gen_sari_i32(ret
, ret
, ofs
);
768 if (TCG_TARGET_HAS_ext16s_i32
) {
769 tcg_gen_shri_i32(ret
, arg
, ofs
);
770 tcg_gen_ext16s_i32(ret
, ret
);
775 if (TCG_TARGET_HAS_ext8s_i32
) {
776 tcg_gen_shri_i32(ret
, arg
, ofs
);
777 tcg_gen_ext8s_i32(ret
, ret
);
783 tcg_gen_shli_i32(ret
, arg
, 32 - len
- ofs
);
784 tcg_gen_sari_i32(ret
, ret
, 32 - len
);
788 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
789 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
791 void tcg_gen_extract2_i32(TCGv_i32 ret
, TCGv_i32 al
, TCGv_i32 ah
,
794 tcg_debug_assert(ofs
<= 32);
796 tcg_gen_mov_i32(ret
, al
);
797 } else if (ofs
== 32) {
798 tcg_gen_mov_i32(ret
, ah
);
799 } else if (al
== ah
) {
800 tcg_gen_rotri_i32(ret
, al
, ofs
);
801 } else if (TCG_TARGET_HAS_extract2_i32
) {
802 tcg_gen_op4i_i32(INDEX_op_extract2_i32
, ret
, al
, ah
, ofs
);
804 TCGv_i32 t0
= tcg_temp_new_i32();
805 tcg_gen_shri_i32(t0
, al
, ofs
);
806 tcg_gen_deposit_i32(ret
, t0
, ah
, 32 - ofs
, ofs
);
807 tcg_temp_free_i32(t0
);
811 void tcg_gen_movcond_i32(TCGCond cond
, TCGv_i32 ret
, TCGv_i32 c1
,
812 TCGv_i32 c2
, TCGv_i32 v1
, TCGv_i32 v2
)
814 if (cond
== TCG_COND_ALWAYS
) {
815 tcg_gen_mov_i32(ret
, v1
);
816 } else if (cond
== TCG_COND_NEVER
) {
817 tcg_gen_mov_i32(ret
, v2
);
818 } else if (TCG_TARGET_HAS_movcond_i32
) {
819 tcg_gen_op6i_i32(INDEX_op_movcond_i32
, ret
, c1
, c2
, v1
, v2
, cond
);
821 TCGv_i32 t0
= tcg_temp_new_i32();
822 TCGv_i32 t1
= tcg_temp_new_i32();
823 tcg_gen_setcond_i32(cond
, t0
, c1
, c2
);
824 tcg_gen_neg_i32(t0
, t0
);
825 tcg_gen_and_i32(t1
, v1
, t0
);
826 tcg_gen_andc_i32(ret
, v2
, t0
);
827 tcg_gen_or_i32(ret
, ret
, t1
);
828 tcg_temp_free_i32(t0
);
829 tcg_temp_free_i32(t1
);
833 void tcg_gen_add2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 al
,
834 TCGv_i32 ah
, TCGv_i32 bl
, TCGv_i32 bh
)
836 if (TCG_TARGET_HAS_add2_i32
) {
837 tcg_gen_op6_i32(INDEX_op_add2_i32
, rl
, rh
, al
, ah
, bl
, bh
);
839 TCGv_i64 t0
= tcg_temp_new_i64();
840 TCGv_i64 t1
= tcg_temp_new_i64();
841 tcg_gen_concat_i32_i64(t0
, al
, ah
);
842 tcg_gen_concat_i32_i64(t1
, bl
, bh
);
843 tcg_gen_add_i64(t0
, t0
, t1
);
844 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
845 tcg_temp_free_i64(t0
);
846 tcg_temp_free_i64(t1
);
850 void tcg_gen_sub2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 al
,
851 TCGv_i32 ah
, TCGv_i32 bl
, TCGv_i32 bh
)
853 if (TCG_TARGET_HAS_sub2_i32
) {
854 tcg_gen_op6_i32(INDEX_op_sub2_i32
, rl
, rh
, al
, ah
, bl
, bh
);
856 TCGv_i64 t0
= tcg_temp_new_i64();
857 TCGv_i64 t1
= tcg_temp_new_i64();
858 tcg_gen_concat_i32_i64(t0
, al
, ah
);
859 tcg_gen_concat_i32_i64(t1
, bl
, bh
);
860 tcg_gen_sub_i64(t0
, t0
, t1
);
861 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
862 tcg_temp_free_i64(t0
);
863 tcg_temp_free_i64(t1
);
867 void tcg_gen_mulu2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
869 if (TCG_TARGET_HAS_mulu2_i32
) {
870 tcg_gen_op4_i32(INDEX_op_mulu2_i32
, rl
, rh
, arg1
, arg2
);
871 } else if (TCG_TARGET_HAS_muluh_i32
) {
872 TCGv_i32 t
= tcg_temp_new_i32();
873 tcg_gen_op3_i32(INDEX_op_mul_i32
, t
, arg1
, arg2
);
874 tcg_gen_op3_i32(INDEX_op_muluh_i32
, rh
, arg1
, arg2
);
875 tcg_gen_mov_i32(rl
, t
);
876 tcg_temp_free_i32(t
);
878 TCGv_i64 t0
= tcg_temp_new_i64();
879 TCGv_i64 t1
= tcg_temp_new_i64();
880 tcg_gen_extu_i32_i64(t0
, arg1
);
881 tcg_gen_extu_i32_i64(t1
, arg2
);
882 tcg_gen_mul_i64(t0
, t0
, t1
);
883 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
884 tcg_temp_free_i64(t0
);
885 tcg_temp_free_i64(t1
);
889 void tcg_gen_muls2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
891 if (TCG_TARGET_HAS_muls2_i32
) {
892 tcg_gen_op4_i32(INDEX_op_muls2_i32
, rl
, rh
, arg1
, arg2
);
893 } else if (TCG_TARGET_HAS_mulsh_i32
) {
894 TCGv_i32 t
= tcg_temp_new_i32();
895 tcg_gen_op3_i32(INDEX_op_mul_i32
, t
, arg1
, arg2
);
896 tcg_gen_op3_i32(INDEX_op_mulsh_i32
, rh
, arg1
, arg2
);
897 tcg_gen_mov_i32(rl
, t
);
898 tcg_temp_free_i32(t
);
899 } else if (TCG_TARGET_REG_BITS
== 32) {
900 TCGv_i32 t0
= tcg_temp_new_i32();
901 TCGv_i32 t1
= tcg_temp_new_i32();
902 TCGv_i32 t2
= tcg_temp_new_i32();
903 TCGv_i32 t3
= tcg_temp_new_i32();
904 tcg_gen_mulu2_i32(t0
, t1
, arg1
, arg2
);
905 /* Adjust for negative inputs. */
906 tcg_gen_sari_i32(t2
, arg1
, 31);
907 tcg_gen_sari_i32(t3
, arg2
, 31);
908 tcg_gen_and_i32(t2
, t2
, arg2
);
909 tcg_gen_and_i32(t3
, t3
, arg1
);
910 tcg_gen_sub_i32(rh
, t1
, t2
);
911 tcg_gen_sub_i32(rh
, rh
, t3
);
912 tcg_gen_mov_i32(rl
, t0
);
913 tcg_temp_free_i32(t0
);
914 tcg_temp_free_i32(t1
);
915 tcg_temp_free_i32(t2
);
916 tcg_temp_free_i32(t3
);
918 TCGv_i64 t0
= tcg_temp_new_i64();
919 TCGv_i64 t1
= tcg_temp_new_i64();
920 tcg_gen_ext_i32_i64(t0
, arg1
);
921 tcg_gen_ext_i32_i64(t1
, arg2
);
922 tcg_gen_mul_i64(t0
, t0
, t1
);
923 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
924 tcg_temp_free_i64(t0
);
925 tcg_temp_free_i64(t1
);
929 void tcg_gen_mulsu2_i32(TCGv_i32 rl
, TCGv_i32 rh
, TCGv_i32 arg1
, TCGv_i32 arg2
)
931 if (TCG_TARGET_REG_BITS
== 32) {
932 TCGv_i32 t0
= tcg_temp_new_i32();
933 TCGv_i32 t1
= tcg_temp_new_i32();
934 TCGv_i32 t2
= tcg_temp_new_i32();
935 tcg_gen_mulu2_i32(t0
, t1
, arg1
, arg2
);
936 /* Adjust for negative input for the signed arg1. */
937 tcg_gen_sari_i32(t2
, arg1
, 31);
938 tcg_gen_and_i32(t2
, t2
, arg2
);
939 tcg_gen_sub_i32(rh
, t1
, t2
);
940 tcg_gen_mov_i32(rl
, t0
);
941 tcg_temp_free_i32(t0
);
942 tcg_temp_free_i32(t1
);
943 tcg_temp_free_i32(t2
);
945 TCGv_i64 t0
= tcg_temp_new_i64();
946 TCGv_i64 t1
= tcg_temp_new_i64();
947 tcg_gen_ext_i32_i64(t0
, arg1
);
948 tcg_gen_extu_i32_i64(t1
, arg2
);
949 tcg_gen_mul_i64(t0
, t0
, t1
);
950 tcg_gen_extr_i64_i32(rl
, rh
, t0
);
951 tcg_temp_free_i64(t0
);
952 tcg_temp_free_i64(t1
);
956 void tcg_gen_ext8s_i32(TCGv_i32 ret
, TCGv_i32 arg
)
958 if (TCG_TARGET_HAS_ext8s_i32
) {
959 tcg_gen_op2_i32(INDEX_op_ext8s_i32
, ret
, arg
);
961 tcg_gen_shli_i32(ret
, arg
, 24);
962 tcg_gen_sari_i32(ret
, ret
, 24);
966 void tcg_gen_ext16s_i32(TCGv_i32 ret
, TCGv_i32 arg
)
968 if (TCG_TARGET_HAS_ext16s_i32
) {
969 tcg_gen_op2_i32(INDEX_op_ext16s_i32
, ret
, arg
);
971 tcg_gen_shli_i32(ret
, arg
, 16);
972 tcg_gen_sari_i32(ret
, ret
, 16);
976 void tcg_gen_ext8u_i32(TCGv_i32 ret
, TCGv_i32 arg
)
978 if (TCG_TARGET_HAS_ext8u_i32
) {
979 tcg_gen_op2_i32(INDEX_op_ext8u_i32
, ret
, arg
);
981 tcg_gen_andi_i32(ret
, arg
, 0xffu
);
985 void tcg_gen_ext16u_i32(TCGv_i32 ret
, TCGv_i32 arg
)
987 if (TCG_TARGET_HAS_ext16u_i32
) {
988 tcg_gen_op2_i32(INDEX_op_ext16u_i32
, ret
, arg
);
990 tcg_gen_andi_i32(ret
, arg
, 0xffffu
);
994 void tcg_gen_bswap16_i32(TCGv_i32 ret
, TCGv_i32 arg
, int flags
)
996 /* Only one extension flag may be present. */
997 tcg_debug_assert(!(flags
& TCG_BSWAP_OS
) || !(flags
& TCG_BSWAP_OZ
));
999 if (TCG_TARGET_HAS_bswap16_i32
) {
1000 tcg_gen_op3i_i32(INDEX_op_bswap16_i32
, ret
, arg
, flags
);
1002 TCGv_i32 t0
= tcg_temp_new_i32();
1003 TCGv_i32 t1
= tcg_temp_new_i32();
1005 tcg_gen_shri_i32(t0
, arg
, 8);
1006 if (!(flags
& TCG_BSWAP_IZ
)) {
1007 tcg_gen_ext8u_i32(t0
, t0
);
1010 if (flags
& TCG_BSWAP_OS
) {
1011 tcg_gen_shli_i32(t1
, arg
, 24);
1012 tcg_gen_sari_i32(t1
, t1
, 16);
1013 } else if (flags
& TCG_BSWAP_OZ
) {
1014 tcg_gen_ext8u_i32(t1
, arg
);
1015 tcg_gen_shli_i32(t1
, t1
, 8);
1017 tcg_gen_shli_i32(t1
, arg
, 8);
1020 tcg_gen_or_i32(ret
, t0
, t1
);
1021 tcg_temp_free_i32(t0
);
1022 tcg_temp_free_i32(t1
);
1026 void tcg_gen_bswap32_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1028 if (TCG_TARGET_HAS_bswap32_i32
) {
1029 tcg_gen_op3i_i32(INDEX_op_bswap32_i32
, ret
, arg
, 0);
1031 TCGv_i32 t0
= tcg_temp_new_i32();
1032 TCGv_i32 t1
= tcg_temp_new_i32();
1033 TCGv_i32 t2
= tcg_constant_i32(0x00ff00ff);
1036 tcg_gen_shri_i32(t0
, arg
, 8); /* t0 = .abc */
1037 tcg_gen_and_i32(t1
, arg
, t2
); /* t1 = .b.d */
1038 tcg_gen_and_i32(t0
, t0
, t2
); /* t0 = .a.c */
1039 tcg_gen_shli_i32(t1
, t1
, 8); /* t1 = b.d. */
1040 tcg_gen_or_i32(ret
, t0
, t1
); /* ret = badc */
1042 tcg_gen_shri_i32(t0
, ret
, 16); /* t0 = ..ba */
1043 tcg_gen_shli_i32(t1
, ret
, 16); /* t1 = dc.. */
1044 tcg_gen_or_i32(ret
, t0
, t1
); /* ret = dcba */
1046 tcg_temp_free_i32(t0
);
1047 tcg_temp_free_i32(t1
);
1051 void tcg_gen_hswap_i32(TCGv_i32 ret
, TCGv_i32 arg
)
1053 /* Swapping 2 16-bit elements is a rotate. */
1054 tcg_gen_rotli_i32(ret
, arg
, 16);
1057 void tcg_gen_smin_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1059 tcg_gen_movcond_i32(TCG_COND_LT
, ret
, a
, b
, a
, b
);
1062 void tcg_gen_umin_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1064 tcg_gen_movcond_i32(TCG_COND_LTU
, ret
, a
, b
, a
, b
);
1067 void tcg_gen_smax_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1069 tcg_gen_movcond_i32(TCG_COND_LT
, ret
, a
, b
, b
, a
);
1072 void tcg_gen_umax_i32(TCGv_i32 ret
, TCGv_i32 a
, TCGv_i32 b
)
1074 tcg_gen_movcond_i32(TCG_COND_LTU
, ret
, a
, b
, b
, a
);
1077 void tcg_gen_abs_i32(TCGv_i32 ret
, TCGv_i32 a
)
1079 TCGv_i32 t
= tcg_temp_new_i32();
1081 tcg_gen_sari_i32(t
, a
, 31);
1082 tcg_gen_xor_i32(ret
, a
, t
);
1083 tcg_gen_sub_i32(ret
, ret
, t
);
1084 tcg_temp_free_i32(t
);
1089 #if TCG_TARGET_REG_BITS == 32
1090 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1092 void tcg_gen_discard_i64(TCGv_i64 arg
)
1094 tcg_gen_discard_i32(TCGV_LOW(arg
));
1095 tcg_gen_discard_i32(TCGV_HIGH(arg
));
1098 void tcg_gen_mov_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1100 TCGTemp
*ts
= tcgv_i64_temp(arg
);
1102 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1103 if (ts
->kind
== TEMP_CONST
) {
1104 tcg_gen_movi_i64(ret
, ts
->val
);
1106 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1107 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
));
1111 void tcg_gen_movi_i64(TCGv_i64 ret
, int64_t arg
)
1113 tcg_gen_movi_i32(TCGV_LOW(ret
), arg
);
1114 tcg_gen_movi_i32(TCGV_HIGH(ret
), arg
>> 32);
1117 void tcg_gen_ld8u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1119 tcg_gen_ld8u_i32(TCGV_LOW(ret
), arg2
, offset
);
1120 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1123 void tcg_gen_ld8s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1125 tcg_gen_ld8s_i32(TCGV_LOW(ret
), arg2
, offset
);
1126 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1129 void tcg_gen_ld16u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1131 tcg_gen_ld16u_i32(TCGV_LOW(ret
), arg2
, offset
);
1132 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1135 void tcg_gen_ld16s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1137 tcg_gen_ld16s_i32(TCGV_LOW(ret
), arg2
, offset
);
1138 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1141 void tcg_gen_ld32u_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1143 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1144 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1147 void tcg_gen_ld32s_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1149 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1150 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1153 void tcg_gen_ld_i64(TCGv_i64 ret
, TCGv_ptr arg2
, tcg_target_long offset
)
1155 /* Since arg2 and ret have different types,
1156 they cannot be the same temporary */
1158 tcg_gen_ld_i32(TCGV_HIGH(ret
), arg2
, offset
);
1159 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
+ 4);
1161 tcg_gen_ld_i32(TCGV_LOW(ret
), arg2
, offset
);
1162 tcg_gen_ld_i32(TCGV_HIGH(ret
), arg2
, offset
+ 4);
1166 void tcg_gen_st8_i64(TCGv_i64 arg1
, TCGv_ptr arg2
, tcg_target_long offset
)
1168 tcg_gen_st8_i32(TCGV_LOW(arg1
), arg2
, offset
);
1171 void tcg_gen_st16_i64(TCGv_i64 arg1
, TCGv_ptr arg2
, tcg_target_long offset
)
1173 tcg_gen_st16_i32(TCGV_LOW(arg1
), arg2
, offset
);
1176 void tcg_gen_st32_i64(TCGv_i64 arg1
, TCGv_ptr arg2
, tcg_target_long offset
)
1178 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
);
1181 void tcg_gen_st_i64(TCGv_i64 arg1
, TCGv_ptr arg2
, tcg_target_long offset
)
1184 tcg_gen_st_i32(TCGV_HIGH(arg1
), arg2
, offset
);
1185 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
+ 4);
1187 tcg_gen_st_i32(TCGV_LOW(arg1
), arg2
, offset
);
1188 tcg_gen_st_i32(TCGV_HIGH(arg1
), arg2
, offset
+ 4);
1192 void tcg_gen_add_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1194 tcg_gen_add2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
), TCGV_LOW(arg1
),
1195 TCGV_HIGH(arg1
), TCGV_LOW(arg2
), TCGV_HIGH(arg2
));
1198 void tcg_gen_sub_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1200 tcg_gen_sub2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
), TCGV_LOW(arg1
),
1201 TCGV_HIGH(arg1
), TCGV_LOW(arg2
), TCGV_HIGH(arg2
));
1204 void tcg_gen_and_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1206 tcg_gen_and_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1207 tcg_gen_and_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1210 void tcg_gen_or_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1212 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1213 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1216 void tcg_gen_xor_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1218 tcg_gen_xor_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1219 tcg_gen_xor_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1222 void tcg_gen_shl_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1224 gen_helper_shl_i64(ret
, arg1
, arg2
);
1227 void tcg_gen_shr_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1229 gen_helper_shr_i64(ret
, arg1
, arg2
);
1232 void tcg_gen_sar_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1234 gen_helper_sar_i64(ret
, arg1
, arg2
);
1237 void tcg_gen_mul_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1242 t0
= tcg_temp_new_i64();
1243 t1
= tcg_temp_new_i32();
1245 tcg_gen_mulu2_i32(TCGV_LOW(t0
), TCGV_HIGH(t0
),
1246 TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1248 tcg_gen_mul_i32(t1
, TCGV_LOW(arg1
), TCGV_HIGH(arg2
));
1249 tcg_gen_add_i32(TCGV_HIGH(t0
), TCGV_HIGH(t0
), t1
);
1250 tcg_gen_mul_i32(t1
, TCGV_HIGH(arg1
), TCGV_LOW(arg2
));
1251 tcg_gen_add_i32(TCGV_HIGH(t0
), TCGV_HIGH(t0
), t1
);
1253 tcg_gen_mov_i64(ret
, t0
);
1254 tcg_temp_free_i64(t0
);
1255 tcg_temp_free_i32(t1
);
1260 void tcg_gen_movi_i64(TCGv_i64 ret
, int64_t arg
)
1262 tcg_gen_mov_i64(ret
, tcg_constant_i64(arg
));
1265 #endif /* TCG_TARGET_REG_SIZE == 32 */
1267 void tcg_gen_addi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1269 /* some cases can be optimized here */
1271 tcg_gen_mov_i64(ret
, arg1
);
1272 } else if (TCG_TARGET_REG_BITS
== 64) {
1273 tcg_gen_add_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1275 tcg_gen_add2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1276 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1277 tcg_constant_i32(arg2
), tcg_constant_i32(arg2
>> 32));
1281 void tcg_gen_subfi_i64(TCGv_i64 ret
, int64_t arg1
, TCGv_i64 arg2
)
1283 if (arg1
== 0 && TCG_TARGET_HAS_neg_i64
) {
1284 /* Don't recurse with tcg_gen_neg_i64. */
1285 tcg_gen_op2_i64(INDEX_op_neg_i64
, ret
, arg2
);
1286 } else if (TCG_TARGET_REG_BITS
== 64) {
1287 tcg_gen_sub_i64(ret
, tcg_constant_i64(arg1
), arg2
);
1289 tcg_gen_sub2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1290 tcg_constant_i32(arg1
), tcg_constant_i32(arg1
>> 32),
1291 TCGV_LOW(arg2
), TCGV_HIGH(arg2
));
1295 void tcg_gen_subi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1297 /* some cases can be optimized here */
1299 tcg_gen_mov_i64(ret
, arg1
);
1300 } else if (TCG_TARGET_REG_BITS
== 64) {
1301 tcg_gen_sub_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1303 tcg_gen_sub2_i32(TCGV_LOW(ret
), TCGV_HIGH(ret
),
1304 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1305 tcg_constant_i32(arg2
), tcg_constant_i32(arg2
>> 32));
1309 void tcg_gen_andi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1311 if (TCG_TARGET_REG_BITS
== 32) {
1312 tcg_gen_andi_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1313 tcg_gen_andi_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1317 /* Some cases can be optimized here. */
1320 tcg_gen_movi_i64(ret
, 0);
1323 tcg_gen_mov_i64(ret
, arg1
);
1326 /* Don't recurse with tcg_gen_ext8u_i64. */
1327 if (TCG_TARGET_HAS_ext8u_i64
) {
1328 tcg_gen_op2_i64(INDEX_op_ext8u_i64
, ret
, arg1
);
1333 if (TCG_TARGET_HAS_ext16u_i64
) {
1334 tcg_gen_op2_i64(INDEX_op_ext16u_i64
, ret
, arg1
);
1339 if (TCG_TARGET_HAS_ext32u_i64
) {
1340 tcg_gen_op2_i64(INDEX_op_ext32u_i64
, ret
, arg1
);
1346 tcg_gen_and_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1349 void tcg_gen_ori_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1351 if (TCG_TARGET_REG_BITS
== 32) {
1352 tcg_gen_ori_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1353 tcg_gen_ori_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1356 /* Some cases can be optimized here. */
1358 tcg_gen_movi_i64(ret
, -1);
1359 } else if (arg2
== 0) {
1360 tcg_gen_mov_i64(ret
, arg1
);
1362 tcg_gen_or_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1366 void tcg_gen_xori_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1368 if (TCG_TARGET_REG_BITS
== 32) {
1369 tcg_gen_xori_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), arg2
);
1370 tcg_gen_xori_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), arg2
>> 32);
1373 /* Some cases can be optimized here. */
1375 tcg_gen_mov_i64(ret
, arg1
);
1376 } else if (arg2
== -1 && TCG_TARGET_HAS_not_i64
) {
1377 /* Don't recurse with tcg_gen_not_i64. */
1378 tcg_gen_op2_i64(INDEX_op_not_i64
, ret
, arg1
);
1380 tcg_gen_xor_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1384 static inline void tcg_gen_shifti_i64(TCGv_i64 ret
, TCGv_i64 arg1
,
1385 unsigned c
, bool right
, bool arith
)
1387 tcg_debug_assert(c
< 64);
1389 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
1390 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
1391 } else if (c
>= 32) {
1395 tcg_gen_sari_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), c
);
1396 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), 31);
1398 tcg_gen_shri_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), c
);
1399 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1402 tcg_gen_shli_i32(TCGV_HIGH(ret
), TCGV_LOW(arg1
), c
);
1403 tcg_gen_movi_i32(TCGV_LOW(ret
), 0);
1406 if (TCG_TARGET_HAS_extract2_i32
) {
1407 tcg_gen_extract2_i32(TCGV_LOW(ret
),
1408 TCGV_LOW(arg1
), TCGV_HIGH(arg1
), c
);
1410 tcg_gen_shri_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), c
);
1411 tcg_gen_deposit_i32(TCGV_LOW(ret
), TCGV_LOW(ret
),
1412 TCGV_HIGH(arg1
), 32 - c
, c
);
1415 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), c
);
1417 tcg_gen_shri_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), c
);
1420 if (TCG_TARGET_HAS_extract2_i32
) {
1421 tcg_gen_extract2_i32(TCGV_HIGH(ret
),
1422 TCGV_LOW(arg1
), TCGV_HIGH(arg1
), 32 - c
);
1424 TCGv_i32 t0
= tcg_temp_new_i32();
1425 tcg_gen_shri_i32(t0
, TCGV_LOW(arg1
), 32 - c
);
1426 tcg_gen_deposit_i32(TCGV_HIGH(ret
), t0
,
1427 TCGV_HIGH(arg1
), c
, 32 - c
);
1428 tcg_temp_free_i32(t0
);
1430 tcg_gen_shli_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), c
);
1434 void tcg_gen_shli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1436 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1437 if (TCG_TARGET_REG_BITS
== 32) {
1438 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 0, 0);
1439 } else if (arg2
== 0) {
1440 tcg_gen_mov_i64(ret
, arg1
);
1442 tcg_gen_shl_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1446 void tcg_gen_shri_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1448 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1449 if (TCG_TARGET_REG_BITS
== 32) {
1450 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 1, 0);
1451 } else if (arg2
== 0) {
1452 tcg_gen_mov_i64(ret
, arg1
);
1454 tcg_gen_shr_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1458 void tcg_gen_sari_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1460 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
1461 if (TCG_TARGET_REG_BITS
== 32) {
1462 tcg_gen_shifti_i64(ret
, arg1
, arg2
, 1, 1);
1463 } else if (arg2
== 0) {
1464 tcg_gen_mov_i64(ret
, arg1
);
1466 tcg_gen_sar_i64(ret
, arg1
, tcg_constant_i64(arg2
));
1470 void tcg_gen_brcond_i64(TCGCond cond
, TCGv_i64 arg1
, TCGv_i64 arg2
, TCGLabel
*l
)
1472 if (cond
== TCG_COND_ALWAYS
) {
1474 } else if (cond
!= TCG_COND_NEVER
) {
1476 if (TCG_TARGET_REG_BITS
== 32) {
1477 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32
, TCGV_LOW(arg1
),
1478 TCGV_HIGH(arg1
), TCGV_LOW(arg2
),
1479 TCGV_HIGH(arg2
), cond
, label_arg(l
));
1481 tcg_gen_op4ii_i64(INDEX_op_brcond_i64
, arg1
, arg2
, cond
,
1487 void tcg_gen_brcondi_i64(TCGCond cond
, TCGv_i64 arg1
, int64_t arg2
, TCGLabel
*l
)
1489 if (TCG_TARGET_REG_BITS
== 64) {
1490 tcg_gen_brcond_i64(cond
, arg1
, tcg_constant_i64(arg2
), l
);
1491 } else if (cond
== TCG_COND_ALWAYS
) {
1493 } else if (cond
!= TCG_COND_NEVER
) {
1495 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32
,
1496 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1497 tcg_constant_i32(arg2
),
1498 tcg_constant_i32(arg2
>> 32),
1499 cond
, label_arg(l
));
1503 void tcg_gen_setcond_i64(TCGCond cond
, TCGv_i64 ret
,
1504 TCGv_i64 arg1
, TCGv_i64 arg2
)
1506 if (cond
== TCG_COND_ALWAYS
) {
1507 tcg_gen_movi_i64(ret
, 1);
1508 } else if (cond
== TCG_COND_NEVER
) {
1509 tcg_gen_movi_i64(ret
, 0);
1511 if (TCG_TARGET_REG_BITS
== 32) {
1512 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, TCGV_LOW(ret
),
1513 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1514 TCGV_LOW(arg2
), TCGV_HIGH(arg2
), cond
);
1515 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1517 tcg_gen_op4i_i64(INDEX_op_setcond_i64
, ret
, arg1
, arg2
, cond
);
1522 void tcg_gen_setcondi_i64(TCGCond cond
, TCGv_i64 ret
,
1523 TCGv_i64 arg1
, int64_t arg2
)
1525 if (TCG_TARGET_REG_BITS
== 64) {
1526 tcg_gen_setcond_i64(cond
, ret
, arg1
, tcg_constant_i64(arg2
));
1527 } else if (cond
== TCG_COND_ALWAYS
) {
1528 tcg_gen_movi_i64(ret
, 1);
1529 } else if (cond
== TCG_COND_NEVER
) {
1530 tcg_gen_movi_i64(ret
, 0);
1532 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, TCGV_LOW(ret
),
1533 TCGV_LOW(arg1
), TCGV_HIGH(arg1
),
1534 tcg_constant_i32(arg2
),
1535 tcg_constant_i32(arg2
>> 32), cond
);
1536 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1540 void tcg_gen_muli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
1543 tcg_gen_movi_i64(ret
, 0);
1544 } else if (is_power_of_2(arg2
)) {
1545 tcg_gen_shli_i64(ret
, arg1
, ctz64(arg2
));
1547 TCGv_i64 t0
= tcg_const_i64(arg2
);
1548 tcg_gen_mul_i64(ret
, arg1
, t0
);
1549 tcg_temp_free_i64(t0
);
1553 void tcg_gen_div_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1555 if (TCG_TARGET_HAS_div_i64
) {
1556 tcg_gen_op3_i64(INDEX_op_div_i64
, ret
, arg1
, arg2
);
1557 } else if (TCG_TARGET_HAS_div2_i64
) {
1558 TCGv_i64 t0
= tcg_temp_new_i64();
1559 tcg_gen_sari_i64(t0
, arg1
, 63);
1560 tcg_gen_op5_i64(INDEX_op_div2_i64
, ret
, t0
, arg1
, t0
, arg2
);
1561 tcg_temp_free_i64(t0
);
1563 gen_helper_div_i64(ret
, arg1
, arg2
);
1567 void tcg_gen_rem_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1569 if (TCG_TARGET_HAS_rem_i64
) {
1570 tcg_gen_op3_i64(INDEX_op_rem_i64
, ret
, arg1
, arg2
);
1571 } else if (TCG_TARGET_HAS_div_i64
) {
1572 TCGv_i64 t0
= tcg_temp_new_i64();
1573 tcg_gen_op3_i64(INDEX_op_div_i64
, t0
, arg1
, arg2
);
1574 tcg_gen_mul_i64(t0
, t0
, arg2
);
1575 tcg_gen_sub_i64(ret
, arg1
, t0
);
1576 tcg_temp_free_i64(t0
);
1577 } else if (TCG_TARGET_HAS_div2_i64
) {
1578 TCGv_i64 t0
= tcg_temp_new_i64();
1579 tcg_gen_sari_i64(t0
, arg1
, 63);
1580 tcg_gen_op5_i64(INDEX_op_div2_i64
, t0
, ret
, arg1
, t0
, arg2
);
1581 tcg_temp_free_i64(t0
);
1583 gen_helper_rem_i64(ret
, arg1
, arg2
);
1587 void tcg_gen_divu_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1589 if (TCG_TARGET_HAS_div_i64
) {
1590 tcg_gen_op3_i64(INDEX_op_divu_i64
, ret
, arg1
, arg2
);
1591 } else if (TCG_TARGET_HAS_div2_i64
) {
1592 TCGv_i64 t0
= tcg_temp_new_i64();
1593 tcg_gen_movi_i64(t0
, 0);
1594 tcg_gen_op5_i64(INDEX_op_divu2_i64
, ret
, t0
, arg1
, t0
, arg2
);
1595 tcg_temp_free_i64(t0
);
1597 gen_helper_divu_i64(ret
, arg1
, arg2
);
1601 void tcg_gen_remu_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1603 if (TCG_TARGET_HAS_rem_i64
) {
1604 tcg_gen_op3_i64(INDEX_op_remu_i64
, ret
, arg1
, arg2
);
1605 } else if (TCG_TARGET_HAS_div_i64
) {
1606 TCGv_i64 t0
= tcg_temp_new_i64();
1607 tcg_gen_op3_i64(INDEX_op_divu_i64
, t0
, arg1
, arg2
);
1608 tcg_gen_mul_i64(t0
, t0
, arg2
);
1609 tcg_gen_sub_i64(ret
, arg1
, t0
);
1610 tcg_temp_free_i64(t0
);
1611 } else if (TCG_TARGET_HAS_div2_i64
) {
1612 TCGv_i64 t0
= tcg_temp_new_i64();
1613 tcg_gen_movi_i64(t0
, 0);
1614 tcg_gen_op5_i64(INDEX_op_divu2_i64
, t0
, ret
, arg1
, t0
, arg2
);
1615 tcg_temp_free_i64(t0
);
1617 gen_helper_remu_i64(ret
, arg1
, arg2
);
1621 void tcg_gen_ext8s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1623 if (TCG_TARGET_REG_BITS
== 32) {
1624 tcg_gen_ext8s_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1625 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1626 } else if (TCG_TARGET_HAS_ext8s_i64
) {
1627 tcg_gen_op2_i64(INDEX_op_ext8s_i64
, ret
, arg
);
1629 tcg_gen_shli_i64(ret
, arg
, 56);
1630 tcg_gen_sari_i64(ret
, ret
, 56);
1634 void tcg_gen_ext16s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1636 if (TCG_TARGET_REG_BITS
== 32) {
1637 tcg_gen_ext16s_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1638 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1639 } else if (TCG_TARGET_HAS_ext16s_i64
) {
1640 tcg_gen_op2_i64(INDEX_op_ext16s_i64
, ret
, arg
);
1642 tcg_gen_shli_i64(ret
, arg
, 48);
1643 tcg_gen_sari_i64(ret
, ret
, 48);
1647 void tcg_gen_ext32s_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1649 if (TCG_TARGET_REG_BITS
== 32) {
1650 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1651 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1652 } else if (TCG_TARGET_HAS_ext32s_i64
) {
1653 tcg_gen_op2_i64(INDEX_op_ext32s_i64
, ret
, arg
);
1655 tcg_gen_shli_i64(ret
, arg
, 32);
1656 tcg_gen_sari_i64(ret
, ret
, 32);
1660 void tcg_gen_ext8u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1662 if (TCG_TARGET_REG_BITS
== 32) {
1663 tcg_gen_ext8u_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1664 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1665 } else if (TCG_TARGET_HAS_ext8u_i64
) {
1666 tcg_gen_op2_i64(INDEX_op_ext8u_i64
, ret
, arg
);
1668 tcg_gen_andi_i64(ret
, arg
, 0xffu
);
1672 void tcg_gen_ext16u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1674 if (TCG_TARGET_REG_BITS
== 32) {
1675 tcg_gen_ext16u_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1676 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1677 } else if (TCG_TARGET_HAS_ext16u_i64
) {
1678 tcg_gen_op2_i64(INDEX_op_ext16u_i64
, ret
, arg
);
1680 tcg_gen_andi_i64(ret
, arg
, 0xffffu
);
1684 void tcg_gen_ext32u_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1686 if (TCG_TARGET_REG_BITS
== 32) {
1687 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1688 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1689 } else if (TCG_TARGET_HAS_ext32u_i64
) {
1690 tcg_gen_op2_i64(INDEX_op_ext32u_i64
, ret
, arg
);
1692 tcg_gen_andi_i64(ret
, arg
, 0xffffffffu
);
1696 void tcg_gen_bswap16_i64(TCGv_i64 ret
, TCGv_i64 arg
, int flags
)
1698 /* Only one extension flag may be present. */
1699 tcg_debug_assert(!(flags
& TCG_BSWAP_OS
) || !(flags
& TCG_BSWAP_OZ
));
1701 if (TCG_TARGET_REG_BITS
== 32) {
1702 tcg_gen_bswap16_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), flags
);
1703 if (flags
& TCG_BSWAP_OS
) {
1704 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1706 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1708 } else if (TCG_TARGET_HAS_bswap16_i64
) {
1709 tcg_gen_op3i_i64(INDEX_op_bswap16_i64
, ret
, arg
, flags
);
1711 TCGv_i64 t0
= tcg_temp_new_i64();
1712 TCGv_i64 t1
= tcg_temp_new_i64();
1714 tcg_gen_shri_i64(t0
, arg
, 8);
1715 if (!(flags
& TCG_BSWAP_IZ
)) {
1716 tcg_gen_ext8u_i64(t0
, t0
);
1719 if (flags
& TCG_BSWAP_OS
) {
1720 tcg_gen_shli_i64(t1
, arg
, 56);
1721 tcg_gen_sari_i64(t1
, t1
, 48);
1722 } else if (flags
& TCG_BSWAP_OZ
) {
1723 tcg_gen_ext8u_i64(t1
, arg
);
1724 tcg_gen_shli_i64(t1
, t1
, 8);
1726 tcg_gen_shli_i64(t1
, arg
, 8);
1729 tcg_gen_or_i64(ret
, t0
, t1
);
1730 tcg_temp_free_i64(t0
);
1731 tcg_temp_free_i64(t1
);
1735 void tcg_gen_bswap32_i64(TCGv_i64 ret
, TCGv_i64 arg
, int flags
)
1737 /* Only one extension flag may be present. */
1738 tcg_debug_assert(!(flags
& TCG_BSWAP_OS
) || !(flags
& TCG_BSWAP_OZ
));
1740 if (TCG_TARGET_REG_BITS
== 32) {
1741 tcg_gen_bswap32_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1742 if (flags
& TCG_BSWAP_OS
) {
1743 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
1745 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1747 } else if (TCG_TARGET_HAS_bswap32_i64
) {
1748 tcg_gen_op3i_i64(INDEX_op_bswap32_i64
, ret
, arg
, flags
);
1750 TCGv_i64 t0
= tcg_temp_new_i64();
1751 TCGv_i64 t1
= tcg_temp_new_i64();
1752 TCGv_i64 t2
= tcg_constant_i64(0x00ff00ff);
1754 /* arg = xxxxabcd */
1755 tcg_gen_shri_i64(t0
, arg
, 8); /* t0 = .xxxxabc */
1756 tcg_gen_and_i64(t1
, arg
, t2
); /* t1 = .....b.d */
1757 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = .....a.c */
1758 tcg_gen_shli_i64(t1
, t1
, 8); /* t1 = ....b.d. */
1759 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = ....badc */
1761 tcg_gen_shli_i64(t1
, ret
, 48); /* t1 = dc...... */
1762 tcg_gen_shri_i64(t0
, ret
, 16); /* t0 = ......ba */
1763 if (flags
& TCG_BSWAP_OS
) {
1764 tcg_gen_sari_i64(t1
, t1
, 32); /* t1 = ssssdc.. */
1766 tcg_gen_shri_i64(t1
, t1
, 32); /* t1 = ....dc.. */
1768 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = ssssdcba */
1770 tcg_temp_free_i64(t0
);
1771 tcg_temp_free_i64(t1
);
1775 void tcg_gen_bswap64_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1777 if (TCG_TARGET_REG_BITS
== 32) {
1779 t0
= tcg_temp_new_i32();
1780 t1
= tcg_temp_new_i32();
1782 tcg_gen_bswap32_i32(t0
, TCGV_LOW(arg
));
1783 tcg_gen_bswap32_i32(t1
, TCGV_HIGH(arg
));
1784 tcg_gen_mov_i32(TCGV_LOW(ret
), t1
);
1785 tcg_gen_mov_i32(TCGV_HIGH(ret
), t0
);
1786 tcg_temp_free_i32(t0
);
1787 tcg_temp_free_i32(t1
);
1788 } else if (TCG_TARGET_HAS_bswap64_i64
) {
1789 tcg_gen_op3i_i64(INDEX_op_bswap64_i64
, ret
, arg
, 0);
1791 TCGv_i64 t0
= tcg_temp_new_i64();
1792 TCGv_i64 t1
= tcg_temp_new_i64();
1793 TCGv_i64 t2
= tcg_temp_new_i64();
1795 /* arg = abcdefgh */
1796 tcg_gen_movi_i64(t2
, 0x00ff00ff00ff00ffull
);
1797 tcg_gen_shri_i64(t0
, arg
, 8); /* t0 = .abcdefg */
1798 tcg_gen_and_i64(t1
, arg
, t2
); /* t1 = .b.d.f.h */
1799 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = .a.c.e.g */
1800 tcg_gen_shli_i64(t1
, t1
, 8); /* t1 = b.d.f.h. */
1801 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = badcfehg */
1803 tcg_gen_movi_i64(t2
, 0x0000ffff0000ffffull
);
1804 tcg_gen_shri_i64(t0
, ret
, 16); /* t0 = ..badcfe */
1805 tcg_gen_and_i64(t1
, ret
, t2
); /* t1 = ..dc..hg */
1806 tcg_gen_and_i64(t0
, t0
, t2
); /* t0 = ..ba..fe */
1807 tcg_gen_shli_i64(t1
, t1
, 16); /* t1 = dc..hg.. */
1808 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = dcbahgfe */
1810 tcg_gen_shri_i64(t0
, ret
, 32); /* t0 = ....dcba */
1811 tcg_gen_shli_i64(t1
, ret
, 32); /* t1 = hgfe.... */
1812 tcg_gen_or_i64(ret
, t0
, t1
); /* ret = hgfedcba */
1814 tcg_temp_free_i64(t0
);
1815 tcg_temp_free_i64(t1
);
1816 tcg_temp_free_i64(t2
);
1820 void tcg_gen_hswap_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1822 uint64_t m
= 0x0000ffff0000ffffull
;
1823 TCGv_i64 t0
= tcg_temp_new_i64();
1824 TCGv_i64 t1
= tcg_temp_new_i64();
1826 /* See include/qemu/bitops.h, hswap64. */
1827 tcg_gen_rotli_i64(t1
, arg
, 32);
1828 tcg_gen_andi_i64(t0
, t1
, m
);
1829 tcg_gen_shli_i64(t0
, t0
, 16);
1830 tcg_gen_shri_i64(t1
, t1
, 16);
1831 tcg_gen_andi_i64(t1
, t1
, m
);
1832 tcg_gen_or_i64(ret
, t0
, t1
);
1834 tcg_temp_free_i64(t0
);
1835 tcg_temp_free_i64(t1
);
1838 void tcg_gen_wswap_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1840 /* Swapping 2 32-bit elements is a rotate. */
1841 tcg_gen_rotli_i64(ret
, arg
, 32);
1844 void tcg_gen_not_i64(TCGv_i64 ret
, TCGv_i64 arg
)
1846 if (TCG_TARGET_REG_BITS
== 32) {
1847 tcg_gen_not_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
1848 tcg_gen_not_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
));
1849 } else if (TCG_TARGET_HAS_not_i64
) {
1850 tcg_gen_op2_i64(INDEX_op_not_i64
, ret
, arg
);
1852 tcg_gen_xori_i64(ret
, arg
, -1);
1856 void tcg_gen_andc_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1858 if (TCG_TARGET_REG_BITS
== 32) {
1859 tcg_gen_andc_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1860 tcg_gen_andc_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1861 } else if (TCG_TARGET_HAS_andc_i64
) {
1862 tcg_gen_op3_i64(INDEX_op_andc_i64
, ret
, arg1
, arg2
);
1864 TCGv_i64 t0
= tcg_temp_new_i64();
1865 tcg_gen_not_i64(t0
, arg2
);
1866 tcg_gen_and_i64(ret
, arg1
, t0
);
1867 tcg_temp_free_i64(t0
);
1871 void tcg_gen_eqv_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1873 if (TCG_TARGET_REG_BITS
== 32) {
1874 tcg_gen_eqv_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1875 tcg_gen_eqv_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1876 } else if (TCG_TARGET_HAS_eqv_i64
) {
1877 tcg_gen_op3_i64(INDEX_op_eqv_i64
, ret
, arg1
, arg2
);
1879 tcg_gen_xor_i64(ret
, arg1
, arg2
);
1880 tcg_gen_not_i64(ret
, ret
);
1884 void tcg_gen_nand_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1886 if (TCG_TARGET_REG_BITS
== 32) {
1887 tcg_gen_nand_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1888 tcg_gen_nand_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1889 } else if (TCG_TARGET_HAS_nand_i64
) {
1890 tcg_gen_op3_i64(INDEX_op_nand_i64
, ret
, arg1
, arg2
);
1892 tcg_gen_and_i64(ret
, arg1
, arg2
);
1893 tcg_gen_not_i64(ret
, ret
);
1897 void tcg_gen_nor_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1899 if (TCG_TARGET_REG_BITS
== 32) {
1900 tcg_gen_nor_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1901 tcg_gen_nor_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1902 } else if (TCG_TARGET_HAS_nor_i64
) {
1903 tcg_gen_op3_i64(INDEX_op_nor_i64
, ret
, arg1
, arg2
);
1905 tcg_gen_or_i64(ret
, arg1
, arg2
);
1906 tcg_gen_not_i64(ret
, ret
);
1910 void tcg_gen_orc_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1912 if (TCG_TARGET_REG_BITS
== 32) {
1913 tcg_gen_orc_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), TCGV_LOW(arg2
));
1914 tcg_gen_orc_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
), TCGV_HIGH(arg2
));
1915 } else if (TCG_TARGET_HAS_orc_i64
) {
1916 tcg_gen_op3_i64(INDEX_op_orc_i64
, ret
, arg1
, arg2
);
1918 TCGv_i64 t0
= tcg_temp_new_i64();
1919 tcg_gen_not_i64(t0
, arg2
);
1920 tcg_gen_or_i64(ret
, arg1
, t0
);
1921 tcg_temp_free_i64(t0
);
1925 void tcg_gen_clz_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1927 if (TCG_TARGET_HAS_clz_i64
) {
1928 tcg_gen_op3_i64(INDEX_op_clz_i64
, ret
, arg1
, arg2
);
1930 gen_helper_clz_i64(ret
, arg1
, arg2
);
1934 void tcg_gen_clzi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, uint64_t arg2
)
1936 if (TCG_TARGET_REG_BITS
== 32
1937 && TCG_TARGET_HAS_clz_i32
1938 && arg2
<= 0xffffffffu
) {
1939 TCGv_i32 t
= tcg_temp_new_i32();
1940 tcg_gen_clzi_i32(t
, TCGV_LOW(arg1
), arg2
- 32);
1941 tcg_gen_addi_i32(t
, t
, 32);
1942 tcg_gen_clz_i32(TCGV_LOW(ret
), TCGV_HIGH(arg1
), t
);
1943 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1944 tcg_temp_free_i32(t
);
1946 TCGv_i64 t0
= tcg_const_i64(arg2
);
1947 tcg_gen_clz_i64(ret
, arg1
, t0
);
1948 tcg_temp_free_i64(t0
);
1952 void tcg_gen_ctz_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
1954 if (TCG_TARGET_HAS_ctz_i64
) {
1955 tcg_gen_op3_i64(INDEX_op_ctz_i64
, ret
, arg1
, arg2
);
1956 } else if (TCG_TARGET_HAS_ctpop_i64
|| TCG_TARGET_HAS_clz_i64
) {
1957 TCGv_i64 z
, t
= tcg_temp_new_i64();
1959 if (TCG_TARGET_HAS_ctpop_i64
) {
1960 tcg_gen_subi_i64(t
, arg1
, 1);
1961 tcg_gen_andc_i64(t
, t
, arg1
);
1962 tcg_gen_ctpop_i64(t
, t
);
1964 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1965 tcg_gen_neg_i64(t
, arg1
);
1966 tcg_gen_and_i64(t
, t
, arg1
);
1967 tcg_gen_clzi_i64(t
, t
, 64);
1968 tcg_gen_xori_i64(t
, t
, 63);
1970 z
= tcg_constant_i64(0);
1971 tcg_gen_movcond_i64(TCG_COND_EQ
, ret
, arg1
, z
, arg2
, t
);
1972 tcg_temp_free_i64(t
);
1973 tcg_temp_free_i64(z
);
1975 gen_helper_ctz_i64(ret
, arg1
, arg2
);
1979 void tcg_gen_ctzi_i64(TCGv_i64 ret
, TCGv_i64 arg1
, uint64_t arg2
)
1981 if (TCG_TARGET_REG_BITS
== 32
1982 && TCG_TARGET_HAS_ctz_i32
1983 && arg2
<= 0xffffffffu
) {
1984 TCGv_i32 t32
= tcg_temp_new_i32();
1985 tcg_gen_ctzi_i32(t32
, TCGV_HIGH(arg1
), arg2
- 32);
1986 tcg_gen_addi_i32(t32
, t32
, 32);
1987 tcg_gen_ctz_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
), t32
);
1988 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
1989 tcg_temp_free_i32(t32
);
1990 } else if (!TCG_TARGET_HAS_ctz_i64
1991 && TCG_TARGET_HAS_ctpop_i64
1993 /* This equivalence has the advantage of not requiring a fixup. */
1994 TCGv_i64 t
= tcg_temp_new_i64();
1995 tcg_gen_subi_i64(t
, arg1
, 1);
1996 tcg_gen_andc_i64(t
, t
, arg1
);
1997 tcg_gen_ctpop_i64(ret
, t
);
1998 tcg_temp_free_i64(t
);
2000 TCGv_i64 t0
= tcg_const_i64(arg2
);
2001 tcg_gen_ctz_i64(ret
, arg1
, t0
);
2002 tcg_temp_free_i64(t0
);
2006 void tcg_gen_clrsb_i64(TCGv_i64 ret
, TCGv_i64 arg
)
2008 if (TCG_TARGET_HAS_clz_i64
|| TCG_TARGET_HAS_clz_i32
) {
2009 TCGv_i64 t
= tcg_temp_new_i64();
2010 tcg_gen_sari_i64(t
, arg
, 63);
2011 tcg_gen_xor_i64(t
, t
, arg
);
2012 tcg_gen_clzi_i64(t
, t
, 64);
2013 tcg_gen_subi_i64(ret
, t
, 1);
2014 tcg_temp_free_i64(t
);
2016 gen_helper_clrsb_i64(ret
, arg
);
2020 void tcg_gen_ctpop_i64(TCGv_i64 ret
, TCGv_i64 arg1
)
2022 if (TCG_TARGET_HAS_ctpop_i64
) {
2023 tcg_gen_op2_i64(INDEX_op_ctpop_i64
, ret
, arg1
);
2024 } else if (TCG_TARGET_REG_BITS
== 32 && TCG_TARGET_HAS_ctpop_i32
) {
2025 tcg_gen_ctpop_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
2026 tcg_gen_ctpop_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
2027 tcg_gen_add_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), TCGV_HIGH(ret
));
2028 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2030 gen_helper_ctpop_i64(ret
, arg1
);
2034 void tcg_gen_rotl_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2036 if (TCG_TARGET_HAS_rot_i64
) {
2037 tcg_gen_op3_i64(INDEX_op_rotl_i64
, ret
, arg1
, arg2
);
2040 t0
= tcg_temp_new_i64();
2041 t1
= tcg_temp_new_i64();
2042 tcg_gen_shl_i64(t0
, arg1
, arg2
);
2043 tcg_gen_subfi_i64(t1
, 64, arg2
);
2044 tcg_gen_shr_i64(t1
, arg1
, t1
);
2045 tcg_gen_or_i64(ret
, t0
, t1
);
2046 tcg_temp_free_i64(t0
);
2047 tcg_temp_free_i64(t1
);
2051 void tcg_gen_rotli_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
2053 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
2054 /* some cases can be optimized here */
2056 tcg_gen_mov_i64(ret
, arg1
);
2057 } else if (TCG_TARGET_HAS_rot_i64
) {
2058 tcg_gen_rotl_i64(ret
, arg1
, tcg_constant_i64(arg2
));
2061 t0
= tcg_temp_new_i64();
2062 t1
= tcg_temp_new_i64();
2063 tcg_gen_shli_i64(t0
, arg1
, arg2
);
2064 tcg_gen_shri_i64(t1
, arg1
, 64 - arg2
);
2065 tcg_gen_or_i64(ret
, t0
, t1
);
2066 tcg_temp_free_i64(t0
);
2067 tcg_temp_free_i64(t1
);
2071 void tcg_gen_rotr_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2073 if (TCG_TARGET_HAS_rot_i64
) {
2074 tcg_gen_op3_i64(INDEX_op_rotr_i64
, ret
, arg1
, arg2
);
2077 t0
= tcg_temp_new_i64();
2078 t1
= tcg_temp_new_i64();
2079 tcg_gen_shr_i64(t0
, arg1
, arg2
);
2080 tcg_gen_subfi_i64(t1
, 64, arg2
);
2081 tcg_gen_shl_i64(t1
, arg1
, t1
);
2082 tcg_gen_or_i64(ret
, t0
, t1
);
2083 tcg_temp_free_i64(t0
);
2084 tcg_temp_free_i64(t1
);
2088 void tcg_gen_rotri_i64(TCGv_i64 ret
, TCGv_i64 arg1
, int64_t arg2
)
2090 tcg_debug_assert(arg2
>= 0 && arg2
< 64);
2091 /* some cases can be optimized here */
2093 tcg_gen_mov_i64(ret
, arg1
);
2095 tcg_gen_rotli_i64(ret
, arg1
, 64 - arg2
);
2099 void tcg_gen_deposit_i64(TCGv_i64 ret
, TCGv_i64 arg1
, TCGv_i64 arg2
,
2100 unsigned int ofs
, unsigned int len
)
2105 tcg_debug_assert(ofs
< 64);
2106 tcg_debug_assert(len
> 0);
2107 tcg_debug_assert(len
<= 64);
2108 tcg_debug_assert(ofs
+ len
<= 64);
2111 tcg_gen_mov_i64(ret
, arg2
);
2114 if (TCG_TARGET_HAS_deposit_i64
&& TCG_TARGET_deposit_i64_valid(ofs
, len
)) {
2115 tcg_gen_op5ii_i64(INDEX_op_deposit_i64
, ret
, arg1
, arg2
, ofs
, len
);
2119 if (TCG_TARGET_REG_BITS
== 32) {
2121 tcg_gen_deposit_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
),
2122 TCGV_LOW(arg2
), ofs
- 32, len
);
2123 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
));
2126 if (ofs
+ len
<= 32) {
2127 tcg_gen_deposit_i32(TCGV_LOW(ret
), TCGV_LOW(arg1
),
2128 TCGV_LOW(arg2
), ofs
, len
);
2129 tcg_gen_mov_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg1
));
2134 t1
= tcg_temp_new_i64();
2136 if (TCG_TARGET_HAS_extract2_i64
) {
2137 if (ofs
+ len
== 64) {
2138 tcg_gen_shli_i64(t1
, arg1
, len
);
2139 tcg_gen_extract2_i64(ret
, t1
, arg2
, len
);
2143 tcg_gen_extract2_i64(ret
, arg1
, arg2
, len
);
2144 tcg_gen_rotli_i64(ret
, ret
, len
);
2149 mask
= (1ull << len
) - 1;
2150 if (ofs
+ len
< 64) {
2151 tcg_gen_andi_i64(t1
, arg2
, mask
);
2152 tcg_gen_shli_i64(t1
, t1
, ofs
);
2154 tcg_gen_shli_i64(t1
, arg2
, ofs
);
2156 tcg_gen_andi_i64(ret
, arg1
, ~(mask
<< ofs
));
2157 tcg_gen_or_i64(ret
, ret
, t1
);
2159 tcg_temp_free_i64(t1
);
2162 void tcg_gen_deposit_z_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2163 unsigned int ofs
, unsigned int len
)
2165 tcg_debug_assert(ofs
< 64);
2166 tcg_debug_assert(len
> 0);
2167 tcg_debug_assert(len
<= 64);
2168 tcg_debug_assert(ofs
+ len
<= 64);
2170 if (ofs
+ len
== 64) {
2171 tcg_gen_shli_i64(ret
, arg
, ofs
);
2172 } else if (ofs
== 0) {
2173 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2174 } else if (TCG_TARGET_HAS_deposit_i64
2175 && TCG_TARGET_deposit_i64_valid(ofs
, len
)) {
2176 TCGv_i64 zero
= tcg_constant_i64(0);
2177 tcg_gen_op5ii_i64(INDEX_op_deposit_i64
, ret
, zero
, arg
, ofs
, len
);
2179 if (TCG_TARGET_REG_BITS
== 32) {
2181 tcg_gen_deposit_z_i32(TCGV_HIGH(ret
), TCGV_LOW(arg
),
2183 tcg_gen_movi_i32(TCGV_LOW(ret
), 0);
2186 if (ofs
+ len
<= 32) {
2187 tcg_gen_deposit_z_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2188 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2192 /* To help two-operand hosts we prefer to zero-extend first,
2193 which allows ARG to stay live. */
2196 if (TCG_TARGET_HAS_ext32u_i64
) {
2197 tcg_gen_ext32u_i64(ret
, arg
);
2198 tcg_gen_shli_i64(ret
, ret
, ofs
);
2203 if (TCG_TARGET_HAS_ext16u_i64
) {
2204 tcg_gen_ext16u_i64(ret
, arg
);
2205 tcg_gen_shli_i64(ret
, ret
, ofs
);
2210 if (TCG_TARGET_HAS_ext8u_i64
) {
2211 tcg_gen_ext8u_i64(ret
, arg
);
2212 tcg_gen_shli_i64(ret
, ret
, ofs
);
2217 /* Otherwise prefer zero-extension over AND for code size. */
2218 switch (ofs
+ len
) {
2220 if (TCG_TARGET_HAS_ext32u_i64
) {
2221 tcg_gen_shli_i64(ret
, arg
, ofs
);
2222 tcg_gen_ext32u_i64(ret
, ret
);
2227 if (TCG_TARGET_HAS_ext16u_i64
) {
2228 tcg_gen_shli_i64(ret
, arg
, ofs
);
2229 tcg_gen_ext16u_i64(ret
, ret
);
2234 if (TCG_TARGET_HAS_ext8u_i64
) {
2235 tcg_gen_shli_i64(ret
, arg
, ofs
);
2236 tcg_gen_ext8u_i64(ret
, ret
);
2241 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2242 tcg_gen_shli_i64(ret
, ret
, ofs
);
2246 void tcg_gen_extract_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2247 unsigned int ofs
, unsigned int len
)
2249 tcg_debug_assert(ofs
< 64);
2250 tcg_debug_assert(len
> 0);
2251 tcg_debug_assert(len
<= 64);
2252 tcg_debug_assert(ofs
+ len
<= 64);
2254 /* Canonicalize certain special cases, even if extract is supported. */
2255 if (ofs
+ len
== 64) {
2256 tcg_gen_shri_i64(ret
, arg
, 64 - len
);
2260 tcg_gen_andi_i64(ret
, arg
, (1ull << len
) - 1);
2264 if (TCG_TARGET_REG_BITS
== 32) {
2265 /* Look for a 32-bit extract within one of the two words. */
2267 tcg_gen_extract_i32(TCGV_LOW(ret
), TCGV_HIGH(arg
), ofs
- 32, len
);
2268 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2271 if (ofs
+ len
<= 32) {
2272 tcg_gen_extract_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2273 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2276 /* The field is split across two words. One double-word
2277 shift is better than two double-word shifts. */
2281 if (TCG_TARGET_HAS_extract_i64
2282 && TCG_TARGET_extract_i64_valid(ofs
, len
)) {
2283 tcg_gen_op4ii_i64(INDEX_op_extract_i64
, ret
, arg
, ofs
, len
);
2287 /* Assume that zero-extension, if available, is cheaper than a shift. */
2288 switch (ofs
+ len
) {
2290 if (TCG_TARGET_HAS_ext32u_i64
) {
2291 tcg_gen_ext32u_i64(ret
, arg
);
2292 tcg_gen_shri_i64(ret
, ret
, ofs
);
2297 if (TCG_TARGET_HAS_ext16u_i64
) {
2298 tcg_gen_ext16u_i64(ret
, arg
);
2299 tcg_gen_shri_i64(ret
, ret
, ofs
);
2304 if (TCG_TARGET_HAS_ext8u_i64
) {
2305 tcg_gen_ext8u_i64(ret
, arg
);
2306 tcg_gen_shri_i64(ret
, ret
, ofs
);
2312 /* ??? Ideally we'd know what values are available for immediate AND.
2313 Assume that 8 bits are available, plus the special cases of 16 and 32,
2314 so that we get ext8u, ext16u, and ext32u. */
2316 case 1 ... 8: case 16: case 32:
2318 tcg_gen_shri_i64(ret
, arg
, ofs
);
2319 tcg_gen_andi_i64(ret
, ret
, (1ull << len
) - 1);
2322 tcg_gen_shli_i64(ret
, arg
, 64 - len
- ofs
);
2323 tcg_gen_shri_i64(ret
, ret
, 64 - len
);
2328 void tcg_gen_sextract_i64(TCGv_i64 ret
, TCGv_i64 arg
,
2329 unsigned int ofs
, unsigned int len
)
2331 tcg_debug_assert(ofs
< 64);
2332 tcg_debug_assert(len
> 0);
2333 tcg_debug_assert(len
<= 64);
2334 tcg_debug_assert(ofs
+ len
<= 64);
2336 /* Canonicalize certain special cases, even if sextract is supported. */
2337 if (ofs
+ len
== 64) {
2338 tcg_gen_sari_i64(ret
, arg
, 64 - len
);
2344 tcg_gen_ext32s_i64(ret
, arg
);
2347 tcg_gen_ext16s_i64(ret
, arg
);
2350 tcg_gen_ext8s_i64(ret
, arg
);
2355 if (TCG_TARGET_REG_BITS
== 32) {
2356 /* Look for a 32-bit extract within one of the two words. */
2358 tcg_gen_sextract_i32(TCGV_LOW(ret
), TCGV_HIGH(arg
), ofs
- 32, len
);
2359 } else if (ofs
+ len
<= 32) {
2360 tcg_gen_sextract_i32(TCGV_LOW(ret
), TCGV_LOW(arg
), ofs
, len
);
2361 } else if (ofs
== 0) {
2362 tcg_gen_mov_i32(TCGV_LOW(ret
), TCGV_LOW(arg
));
2363 tcg_gen_sextract_i32(TCGV_HIGH(ret
), TCGV_HIGH(arg
), 0, len
- 32);
2365 } else if (len
> 32) {
2366 TCGv_i32 t
= tcg_temp_new_i32();
2367 /* Extract the bits for the high word normally. */
2368 tcg_gen_sextract_i32(t
, TCGV_HIGH(arg
), ofs
+ 32, len
- 32);
2369 /* Shift the field down for the low part. */
2370 tcg_gen_shri_i64(ret
, arg
, ofs
);
2371 /* Overwrite the shift into the high part. */
2372 tcg_gen_mov_i32(TCGV_HIGH(ret
), t
);
2373 tcg_temp_free_i32(t
);
2376 /* Shift the field down for the low part, such that the
2377 field sits at the MSB. */
2378 tcg_gen_shri_i64(ret
, arg
, ofs
+ len
- 32);
2379 /* Shift the field down from the MSB, sign extending. */
2380 tcg_gen_sari_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), 32 - len
);
2382 /* Sign-extend the field from 32 bits. */
2383 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
2387 if (TCG_TARGET_HAS_sextract_i64
2388 && TCG_TARGET_extract_i64_valid(ofs
, len
)) {
2389 tcg_gen_op4ii_i64(INDEX_op_sextract_i64
, ret
, arg
, ofs
, len
);
2393 /* Assume that sign-extension, if available, is cheaper than a shift. */
2394 switch (ofs
+ len
) {
2396 if (TCG_TARGET_HAS_ext32s_i64
) {
2397 tcg_gen_ext32s_i64(ret
, arg
);
2398 tcg_gen_sari_i64(ret
, ret
, ofs
);
2403 if (TCG_TARGET_HAS_ext16s_i64
) {
2404 tcg_gen_ext16s_i64(ret
, arg
);
2405 tcg_gen_sari_i64(ret
, ret
, ofs
);
2410 if (TCG_TARGET_HAS_ext8s_i64
) {
2411 tcg_gen_ext8s_i64(ret
, arg
);
2412 tcg_gen_sari_i64(ret
, ret
, ofs
);
2419 if (TCG_TARGET_HAS_ext32s_i64
) {
2420 tcg_gen_shri_i64(ret
, arg
, ofs
);
2421 tcg_gen_ext32s_i64(ret
, ret
);
2426 if (TCG_TARGET_HAS_ext16s_i64
) {
2427 tcg_gen_shri_i64(ret
, arg
, ofs
);
2428 tcg_gen_ext16s_i64(ret
, ret
);
2433 if (TCG_TARGET_HAS_ext8s_i64
) {
2434 tcg_gen_shri_i64(ret
, arg
, ofs
);
2435 tcg_gen_ext8s_i64(ret
, ret
);
2440 tcg_gen_shli_i64(ret
, arg
, 64 - len
- ofs
);
2441 tcg_gen_sari_i64(ret
, ret
, 64 - len
);
2445 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2446 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2448 void tcg_gen_extract2_i64(TCGv_i64 ret
, TCGv_i64 al
, TCGv_i64 ah
,
2451 tcg_debug_assert(ofs
<= 64);
2453 tcg_gen_mov_i64(ret
, al
);
2454 } else if (ofs
== 64) {
2455 tcg_gen_mov_i64(ret
, ah
);
2456 } else if (al
== ah
) {
2457 tcg_gen_rotri_i64(ret
, al
, ofs
);
2458 } else if (TCG_TARGET_HAS_extract2_i64
) {
2459 tcg_gen_op4i_i64(INDEX_op_extract2_i64
, ret
, al
, ah
, ofs
);
2461 TCGv_i64 t0
= tcg_temp_new_i64();
2462 tcg_gen_shri_i64(t0
, al
, ofs
);
2463 tcg_gen_deposit_i64(ret
, t0
, ah
, 64 - ofs
, ofs
);
2464 tcg_temp_free_i64(t0
);
2468 void tcg_gen_movcond_i64(TCGCond cond
, TCGv_i64 ret
, TCGv_i64 c1
,
2469 TCGv_i64 c2
, TCGv_i64 v1
, TCGv_i64 v2
)
2471 if (cond
== TCG_COND_ALWAYS
) {
2472 tcg_gen_mov_i64(ret
, v1
);
2473 } else if (cond
== TCG_COND_NEVER
) {
2474 tcg_gen_mov_i64(ret
, v2
);
2475 } else if (TCG_TARGET_REG_BITS
== 32) {
2476 TCGv_i32 t0
= tcg_temp_new_i32();
2477 TCGv_i32 t1
= tcg_temp_new_i32();
2478 tcg_gen_op6i_i32(INDEX_op_setcond2_i32
, t0
,
2479 TCGV_LOW(c1
), TCGV_HIGH(c1
),
2480 TCGV_LOW(c2
), TCGV_HIGH(c2
), cond
);
2482 if (TCG_TARGET_HAS_movcond_i32
) {
2483 tcg_gen_movi_i32(t1
, 0);
2484 tcg_gen_movcond_i32(TCG_COND_NE
, TCGV_LOW(ret
), t0
, t1
,
2485 TCGV_LOW(v1
), TCGV_LOW(v2
));
2486 tcg_gen_movcond_i32(TCG_COND_NE
, TCGV_HIGH(ret
), t0
, t1
,
2487 TCGV_HIGH(v1
), TCGV_HIGH(v2
));
2489 tcg_gen_neg_i32(t0
, t0
);
2491 tcg_gen_and_i32(t1
, TCGV_LOW(v1
), t0
);
2492 tcg_gen_andc_i32(TCGV_LOW(ret
), TCGV_LOW(v2
), t0
);
2493 tcg_gen_or_i32(TCGV_LOW(ret
), TCGV_LOW(ret
), t1
);
2495 tcg_gen_and_i32(t1
, TCGV_HIGH(v1
), t0
);
2496 tcg_gen_andc_i32(TCGV_HIGH(ret
), TCGV_HIGH(v2
), t0
);
2497 tcg_gen_or_i32(TCGV_HIGH(ret
), TCGV_HIGH(ret
), t1
);
2499 tcg_temp_free_i32(t0
);
2500 tcg_temp_free_i32(t1
);
2501 } else if (TCG_TARGET_HAS_movcond_i64
) {
2502 tcg_gen_op6i_i64(INDEX_op_movcond_i64
, ret
, c1
, c2
, v1
, v2
, cond
);
2504 TCGv_i64 t0
= tcg_temp_new_i64();
2505 TCGv_i64 t1
= tcg_temp_new_i64();
2506 tcg_gen_setcond_i64(cond
, t0
, c1
, c2
);
2507 tcg_gen_neg_i64(t0
, t0
);
2508 tcg_gen_and_i64(t1
, v1
, t0
);
2509 tcg_gen_andc_i64(ret
, v2
, t0
);
2510 tcg_gen_or_i64(ret
, ret
, t1
);
2511 tcg_temp_free_i64(t0
);
2512 tcg_temp_free_i64(t1
);
2516 void tcg_gen_add2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 al
,
2517 TCGv_i64 ah
, TCGv_i64 bl
, TCGv_i64 bh
)
2519 if (TCG_TARGET_HAS_add2_i64
) {
2520 tcg_gen_op6_i64(INDEX_op_add2_i64
, rl
, rh
, al
, ah
, bl
, bh
);
2522 TCGv_i64 t0
= tcg_temp_new_i64();
2523 TCGv_i64 t1
= tcg_temp_new_i64();
2524 tcg_gen_add_i64(t0
, al
, bl
);
2525 tcg_gen_setcond_i64(TCG_COND_LTU
, t1
, t0
, al
);
2526 tcg_gen_add_i64(rh
, ah
, bh
);
2527 tcg_gen_add_i64(rh
, rh
, t1
);
2528 tcg_gen_mov_i64(rl
, t0
);
2529 tcg_temp_free_i64(t0
);
2530 tcg_temp_free_i64(t1
);
2534 void tcg_gen_sub2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 al
,
2535 TCGv_i64 ah
, TCGv_i64 bl
, TCGv_i64 bh
)
2537 if (TCG_TARGET_HAS_sub2_i64
) {
2538 tcg_gen_op6_i64(INDEX_op_sub2_i64
, rl
, rh
, al
, ah
, bl
, bh
);
2540 TCGv_i64 t0
= tcg_temp_new_i64();
2541 TCGv_i64 t1
= tcg_temp_new_i64();
2542 tcg_gen_sub_i64(t0
, al
, bl
);
2543 tcg_gen_setcond_i64(TCG_COND_LTU
, t1
, al
, bl
);
2544 tcg_gen_sub_i64(rh
, ah
, bh
);
2545 tcg_gen_sub_i64(rh
, rh
, t1
);
2546 tcg_gen_mov_i64(rl
, t0
);
2547 tcg_temp_free_i64(t0
);
2548 tcg_temp_free_i64(t1
);
2552 void tcg_gen_mulu2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2554 if (TCG_TARGET_HAS_mulu2_i64
) {
2555 tcg_gen_op4_i64(INDEX_op_mulu2_i64
, rl
, rh
, arg1
, arg2
);
2556 } else if (TCG_TARGET_HAS_muluh_i64
) {
2557 TCGv_i64 t
= tcg_temp_new_i64();
2558 tcg_gen_op3_i64(INDEX_op_mul_i64
, t
, arg1
, arg2
);
2559 tcg_gen_op3_i64(INDEX_op_muluh_i64
, rh
, arg1
, arg2
);
2560 tcg_gen_mov_i64(rl
, t
);
2561 tcg_temp_free_i64(t
);
2563 TCGv_i64 t0
= tcg_temp_new_i64();
2564 tcg_gen_mul_i64(t0
, arg1
, arg2
);
2565 gen_helper_muluh_i64(rh
, arg1
, arg2
);
2566 tcg_gen_mov_i64(rl
, t0
);
2567 tcg_temp_free_i64(t0
);
2571 void tcg_gen_muls2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2573 if (TCG_TARGET_HAS_muls2_i64
) {
2574 tcg_gen_op4_i64(INDEX_op_muls2_i64
, rl
, rh
, arg1
, arg2
);
2575 } else if (TCG_TARGET_HAS_mulsh_i64
) {
2576 TCGv_i64 t
= tcg_temp_new_i64();
2577 tcg_gen_op3_i64(INDEX_op_mul_i64
, t
, arg1
, arg2
);
2578 tcg_gen_op3_i64(INDEX_op_mulsh_i64
, rh
, arg1
, arg2
);
2579 tcg_gen_mov_i64(rl
, t
);
2580 tcg_temp_free_i64(t
);
2581 } else if (TCG_TARGET_HAS_mulu2_i64
|| TCG_TARGET_HAS_muluh_i64
) {
2582 TCGv_i64 t0
= tcg_temp_new_i64();
2583 TCGv_i64 t1
= tcg_temp_new_i64();
2584 TCGv_i64 t2
= tcg_temp_new_i64();
2585 TCGv_i64 t3
= tcg_temp_new_i64();
2586 tcg_gen_mulu2_i64(t0
, t1
, arg1
, arg2
);
2587 /* Adjust for negative inputs. */
2588 tcg_gen_sari_i64(t2
, arg1
, 63);
2589 tcg_gen_sari_i64(t3
, arg2
, 63);
2590 tcg_gen_and_i64(t2
, t2
, arg2
);
2591 tcg_gen_and_i64(t3
, t3
, arg1
);
2592 tcg_gen_sub_i64(rh
, t1
, t2
);
2593 tcg_gen_sub_i64(rh
, rh
, t3
);
2594 tcg_gen_mov_i64(rl
, t0
);
2595 tcg_temp_free_i64(t0
);
2596 tcg_temp_free_i64(t1
);
2597 tcg_temp_free_i64(t2
);
2598 tcg_temp_free_i64(t3
);
2600 TCGv_i64 t0
= tcg_temp_new_i64();
2601 tcg_gen_mul_i64(t0
, arg1
, arg2
);
2602 gen_helper_mulsh_i64(rh
, arg1
, arg2
);
2603 tcg_gen_mov_i64(rl
, t0
);
2604 tcg_temp_free_i64(t0
);
2608 void tcg_gen_mulsu2_i64(TCGv_i64 rl
, TCGv_i64 rh
, TCGv_i64 arg1
, TCGv_i64 arg2
)
2610 TCGv_i64 t0
= tcg_temp_new_i64();
2611 TCGv_i64 t1
= tcg_temp_new_i64();
2612 TCGv_i64 t2
= tcg_temp_new_i64();
2613 tcg_gen_mulu2_i64(t0
, t1
, arg1
, arg2
);
2614 /* Adjust for negative input for the signed arg1. */
2615 tcg_gen_sari_i64(t2
, arg1
, 63);
2616 tcg_gen_and_i64(t2
, t2
, arg2
);
2617 tcg_gen_sub_i64(rh
, t1
, t2
);
2618 tcg_gen_mov_i64(rl
, t0
);
2619 tcg_temp_free_i64(t0
);
2620 tcg_temp_free_i64(t1
);
2621 tcg_temp_free_i64(t2
);
2624 void tcg_gen_smin_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2626 tcg_gen_movcond_i64(TCG_COND_LT
, ret
, a
, b
, a
, b
);
2629 void tcg_gen_umin_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2631 tcg_gen_movcond_i64(TCG_COND_LTU
, ret
, a
, b
, a
, b
);
2634 void tcg_gen_smax_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2636 tcg_gen_movcond_i64(TCG_COND_LT
, ret
, a
, b
, b
, a
);
2639 void tcg_gen_umax_i64(TCGv_i64 ret
, TCGv_i64 a
, TCGv_i64 b
)
2641 tcg_gen_movcond_i64(TCG_COND_LTU
, ret
, a
, b
, b
, a
);
2644 void tcg_gen_abs_i64(TCGv_i64 ret
, TCGv_i64 a
)
2646 TCGv_i64 t
= tcg_temp_new_i64();
2648 tcg_gen_sari_i64(t
, a
, 63);
2649 tcg_gen_xor_i64(ret
, a
, t
);
2650 tcg_gen_sub_i64(ret
, ret
, t
);
2651 tcg_temp_free_i64(t
);
2654 /* Size changing operations. */
2656 void tcg_gen_extrl_i64_i32(TCGv_i32 ret
, TCGv_i64 arg
)
2658 if (TCG_TARGET_REG_BITS
== 32) {
2659 tcg_gen_mov_i32(ret
, TCGV_LOW(arg
));
2660 } else if (TCG_TARGET_HAS_extrl_i64_i32
) {
2661 tcg_gen_op2(INDEX_op_extrl_i64_i32
,
2662 tcgv_i32_arg(ret
), tcgv_i64_arg(arg
));
2664 tcg_gen_mov_i32(ret
, (TCGv_i32
)arg
);
2668 void tcg_gen_extrh_i64_i32(TCGv_i32 ret
, TCGv_i64 arg
)
2670 if (TCG_TARGET_REG_BITS
== 32) {
2671 tcg_gen_mov_i32(ret
, TCGV_HIGH(arg
));
2672 } else if (TCG_TARGET_HAS_extrh_i64_i32
) {
2673 tcg_gen_op2(INDEX_op_extrh_i64_i32
,
2674 tcgv_i32_arg(ret
), tcgv_i64_arg(arg
));
2676 TCGv_i64 t
= tcg_temp_new_i64();
2677 tcg_gen_shri_i64(t
, arg
, 32);
2678 tcg_gen_mov_i32(ret
, (TCGv_i32
)t
);
2679 tcg_temp_free_i64(t
);
2683 void tcg_gen_extu_i32_i64(TCGv_i64 ret
, TCGv_i32 arg
)
2685 if (TCG_TARGET_REG_BITS
== 32) {
2686 tcg_gen_mov_i32(TCGV_LOW(ret
), arg
);
2687 tcg_gen_movi_i32(TCGV_HIGH(ret
), 0);
2689 tcg_gen_op2(INDEX_op_extu_i32_i64
,
2690 tcgv_i64_arg(ret
), tcgv_i32_arg(arg
));
2694 void tcg_gen_ext_i32_i64(TCGv_i64 ret
, TCGv_i32 arg
)
2696 if (TCG_TARGET_REG_BITS
== 32) {
2697 tcg_gen_mov_i32(TCGV_LOW(ret
), arg
);
2698 tcg_gen_sari_i32(TCGV_HIGH(ret
), TCGV_LOW(ret
), 31);
2700 tcg_gen_op2(INDEX_op_ext_i32_i64
,
2701 tcgv_i64_arg(ret
), tcgv_i32_arg(arg
));
2705 void tcg_gen_concat_i32_i64(TCGv_i64 dest
, TCGv_i32 low
, TCGv_i32 high
)
2709 if (TCG_TARGET_REG_BITS
== 32) {
2710 tcg_gen_mov_i32(TCGV_LOW(dest
), low
);
2711 tcg_gen_mov_i32(TCGV_HIGH(dest
), high
);
2715 tmp
= tcg_temp_new_i64();
2716 /* These extensions are only needed for type correctness.
2717 We may be able to do better given target specific information. */
2718 tcg_gen_extu_i32_i64(tmp
, high
);
2719 tcg_gen_extu_i32_i64(dest
, low
);
2720 /* If deposit is available, use it. Otherwise use the extra
2721 knowledge that we have of the zero-extensions above. */
2722 if (TCG_TARGET_HAS_deposit_i64
&& TCG_TARGET_deposit_i64_valid(32, 32)) {
2723 tcg_gen_deposit_i64(dest
, dest
, tmp
, 32, 32);
2725 tcg_gen_shli_i64(tmp
, tmp
, 32);
2726 tcg_gen_or_i64(dest
, dest
, tmp
);
2728 tcg_temp_free_i64(tmp
);
2731 void tcg_gen_extr_i64_i32(TCGv_i32 lo
, TCGv_i32 hi
, TCGv_i64 arg
)
2733 if (TCG_TARGET_REG_BITS
== 32) {
2734 tcg_gen_mov_i32(lo
, TCGV_LOW(arg
));
2735 tcg_gen_mov_i32(hi
, TCGV_HIGH(arg
));
2737 tcg_gen_extrl_i64_i32(lo
, arg
);
2738 tcg_gen_extrh_i64_i32(hi
, arg
);
2742 void tcg_gen_extr32_i64(TCGv_i64 lo
, TCGv_i64 hi
, TCGv_i64 arg
)
2744 tcg_gen_ext32u_i64(lo
, arg
);
2745 tcg_gen_shri_i64(hi
, arg
, 32);
2748 /* QEMU specific operations. */
2750 void tcg_gen_exit_tb(const TranslationBlock
*tb
, unsigned idx
)
2753 * Let the jit code return the read-only version of the
2754 * TranslationBlock, so that we minimize the pc-relative
2755 * distance of the address of the exit_tb code to TB.
2756 * This will improve utilization of pc-relative address loads.
2758 * TODO: Move this to translator_loop, so that all const
2759 * TranslationBlock pointers refer to read-only memory.
2760 * This requires coordination with targets that do not use
2761 * the translator_loop.
2763 uintptr_t val
= (uintptr_t)tcg_splitwx_to_rx((void *)tb
) + idx
;
2766 tcg_debug_assert(idx
== 0);
2767 } else if (idx
<= TB_EXIT_IDXMAX
) {
2768 #ifdef CONFIG_DEBUG_TCG
2769 /* This is an exit following a goto_tb. Verify that we have
2770 seen this numbered exit before, via tcg_gen_goto_tb. */
2771 tcg_debug_assert(tcg_ctx
->goto_tb_issue_mask
& (1 << idx
));
2774 /* This is an exit via the exitreq label. */
2775 tcg_debug_assert(idx
== TB_EXIT_REQUESTED
);
2778 plugin_gen_disable_mem_helpers();
2779 tcg_gen_op1i(INDEX_op_exit_tb
, val
);
2782 void tcg_gen_goto_tb(unsigned idx
)
2784 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
2785 tcg_debug_assert(!(tcg_ctx
->tb_cflags
& CF_NO_GOTO_TB
));
2786 /* We only support two chained exits. */
2787 tcg_debug_assert(idx
<= TB_EXIT_IDXMAX
);
2788 #ifdef CONFIG_DEBUG_TCG
2789 /* Verify that we haven't seen this numbered exit before. */
2790 tcg_debug_assert((tcg_ctx
->goto_tb_issue_mask
& (1 << idx
)) == 0);
2791 tcg_ctx
->goto_tb_issue_mask
|= 1 << idx
;
2793 plugin_gen_disable_mem_helpers();
2794 tcg_gen_op1i(INDEX_op_goto_tb
, idx
);
2797 void tcg_gen_lookup_and_goto_ptr(void)
2801 if (tcg_ctx
->tb_cflags
& CF_NO_GOTO_PTR
) {
2802 tcg_gen_exit_tb(NULL
, 0);
2806 plugin_gen_disable_mem_helpers();
2807 ptr
= tcg_temp_new_ptr();
2808 gen_helper_lookup_tb_ptr(ptr
, cpu_env
);
2809 tcg_gen_op1i(INDEX_op_goto_ptr
, tcgv_ptr_arg(ptr
));
2810 tcg_temp_free_ptr(ptr
);
2813 static inline MemOp
tcg_canonicalize_memop(MemOp op
, bool is64
, bool st
)
2815 /* Trigger the asserts within as early as possible. */
2816 unsigned a_bits
= get_alignment_bits(op
);
2818 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
2819 if (a_bits
== (op
& MO_SIZE
)) {
2820 op
= (op
& ~MO_AMASK
) | MO_ALIGN
;
2823 switch (op
& MO_SIZE
) {
2841 g_assert_not_reached();
2849 static void gen_ldst_i32(TCGOpcode opc
, TCGv_i32 val
, TCGv addr
,
2850 MemOp memop
, TCGArg idx
)
2852 MemOpIdx oi
= make_memop_idx(memop
, idx
);
2853 #if TARGET_LONG_BITS == 32
2854 tcg_gen_op3i_i32(opc
, val
, addr
, oi
);
2856 if (TCG_TARGET_REG_BITS
== 32) {
2857 tcg_gen_op4i_i32(opc
, val
, TCGV_LOW(addr
), TCGV_HIGH(addr
), oi
);
2859 tcg_gen_op3(opc
, tcgv_i32_arg(val
), tcgv_i64_arg(addr
), oi
);
2864 static void gen_ldst_i64(TCGOpcode opc
, TCGv_i64 val
, TCGv addr
,
2865 MemOp memop
, TCGArg idx
)
2867 MemOpIdx oi
= make_memop_idx(memop
, idx
);
2868 #if TARGET_LONG_BITS == 32
2869 if (TCG_TARGET_REG_BITS
== 32) {
2870 tcg_gen_op4i_i32(opc
, TCGV_LOW(val
), TCGV_HIGH(val
), addr
, oi
);
2872 tcg_gen_op3(opc
, tcgv_i64_arg(val
), tcgv_i32_arg(addr
), oi
);
2875 if (TCG_TARGET_REG_BITS
== 32) {
2876 tcg_gen_op5i_i32(opc
, TCGV_LOW(val
), TCGV_HIGH(val
),
2877 TCGV_LOW(addr
), TCGV_HIGH(addr
), oi
);
2879 tcg_gen_op3i_i64(opc
, val
, addr
, oi
);
2884 static void tcg_gen_req_mo(TCGBar type
)
2886 #ifdef TCG_GUEST_DEFAULT_MO
2887 type
&= TCG_GUEST_DEFAULT_MO
;
2889 type
&= ~TCG_TARGET_DEFAULT_MO
;
2891 tcg_gen_mb(type
| TCG_BAR_SC
);
2895 static inline TCGv
plugin_prep_mem_callbacks(TCGv vaddr
)
2897 #ifdef CONFIG_PLUGIN
2898 if (tcg_ctx
->plugin_insn
!= NULL
) {
2899 /* Save a copy of the vaddr for use after a load. */
2900 TCGv temp
= tcg_temp_new();
2901 tcg_gen_mov_tl(temp
, vaddr
);
2908 static void plugin_gen_mem_callbacks(TCGv vaddr
, MemOpIdx oi
,
2909 enum qemu_plugin_mem_rw rw
)
2911 #ifdef CONFIG_PLUGIN
2912 if (tcg_ctx
->plugin_insn
!= NULL
) {
2913 qemu_plugin_meminfo_t info
= make_plugin_meminfo(oi
, rw
);
2914 plugin_gen_empty_mem_callback(vaddr
, info
);
2915 tcg_temp_free(vaddr
);
2920 void tcg_gen_qemu_ld_i32(TCGv_i32 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2925 tcg_gen_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
2926 memop
= tcg_canonicalize_memop(memop
, 0, 0);
2927 oi
= make_memop_idx(memop
, idx
);
2930 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2932 /* The bswap primitive benefits from zero-extended input. */
2933 if ((memop
& MO_SSIZE
) == MO_SW
) {
2938 addr
= plugin_prep_mem_callbacks(addr
);
2939 gen_ldst_i32(INDEX_op_qemu_ld_i32
, val
, addr
, memop
, idx
);
2940 plugin_gen_mem_callbacks(addr
, oi
, QEMU_PLUGIN_MEM_R
);
2942 if ((orig_memop
^ memop
) & MO_BSWAP
) {
2943 switch (orig_memop
& MO_SIZE
) {
2945 tcg_gen_bswap16_i32(val
, val
, (orig_memop
& MO_SIGN
2946 ? TCG_BSWAP_IZ
| TCG_BSWAP_OS
2947 : TCG_BSWAP_IZ
| TCG_BSWAP_OZ
));
2950 tcg_gen_bswap32_i32(val
, val
);
2953 g_assert_not_reached();
2958 void tcg_gen_qemu_st_i32(TCGv_i32 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
2960 TCGv_i32 swap
= NULL
;
2963 tcg_gen_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
2964 memop
= tcg_canonicalize_memop(memop
, 0, 1);
2965 oi
= make_memop_idx(memop
, idx
);
2967 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
2968 swap
= tcg_temp_new_i32();
2969 switch (memop
& MO_SIZE
) {
2971 tcg_gen_bswap16_i32(swap
, val
, 0);
2974 tcg_gen_bswap32_i32(swap
, val
);
2977 g_assert_not_reached();
2983 addr
= plugin_prep_mem_callbacks(addr
);
2984 if (TCG_TARGET_HAS_qemu_st8_i32
&& (memop
& MO_SIZE
) == MO_8
) {
2985 gen_ldst_i32(INDEX_op_qemu_st8_i32
, val
, addr
, memop
, idx
);
2987 gen_ldst_i32(INDEX_op_qemu_st_i32
, val
, addr
, memop
, idx
);
2989 plugin_gen_mem_callbacks(addr
, oi
, QEMU_PLUGIN_MEM_W
);
2992 tcg_temp_free_i32(swap
);
2996 void tcg_gen_qemu_ld_i64(TCGv_i64 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
3001 if (TCG_TARGET_REG_BITS
== 32 && (memop
& MO_SIZE
) < MO_64
) {
3002 tcg_gen_qemu_ld_i32(TCGV_LOW(val
), addr
, idx
, memop
);
3003 if (memop
& MO_SIGN
) {
3004 tcg_gen_sari_i32(TCGV_HIGH(val
), TCGV_LOW(val
), 31);
3006 tcg_gen_movi_i32(TCGV_HIGH(val
), 0);
3011 tcg_gen_req_mo(TCG_MO_LD_LD
| TCG_MO_ST_LD
);
3012 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3013 oi
= make_memop_idx(memop
, idx
);
3016 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
3018 /* The bswap primitive benefits from zero-extended input. */
3019 if ((memop
& MO_SIGN
) && (memop
& MO_SIZE
) < MO_64
) {
3024 addr
= plugin_prep_mem_callbacks(addr
);
3025 gen_ldst_i64(INDEX_op_qemu_ld_i64
, val
, addr
, memop
, idx
);
3026 plugin_gen_mem_callbacks(addr
, oi
, QEMU_PLUGIN_MEM_R
);
3028 if ((orig_memop
^ memop
) & MO_BSWAP
) {
3029 int flags
= (orig_memop
& MO_SIGN
3030 ? TCG_BSWAP_IZ
| TCG_BSWAP_OS
3031 : TCG_BSWAP_IZ
| TCG_BSWAP_OZ
);
3032 switch (orig_memop
& MO_SIZE
) {
3034 tcg_gen_bswap16_i64(val
, val
, flags
);
3037 tcg_gen_bswap32_i64(val
, val
, flags
);
3040 tcg_gen_bswap64_i64(val
, val
);
3043 g_assert_not_reached();
3048 void tcg_gen_qemu_st_i64(TCGv_i64 val
, TCGv addr
, TCGArg idx
, MemOp memop
)
3050 TCGv_i64 swap
= NULL
;
3053 if (TCG_TARGET_REG_BITS
== 32 && (memop
& MO_SIZE
) < MO_64
) {
3054 tcg_gen_qemu_st_i32(TCGV_LOW(val
), addr
, idx
, memop
);
3058 tcg_gen_req_mo(TCG_MO_LD_ST
| TCG_MO_ST_ST
);
3059 memop
= tcg_canonicalize_memop(memop
, 1, 1);
3060 oi
= make_memop_idx(memop
, idx
);
3062 if (!TCG_TARGET_HAS_MEMORY_BSWAP
&& (memop
& MO_BSWAP
)) {
3063 swap
= tcg_temp_new_i64();
3064 switch (memop
& MO_SIZE
) {
3066 tcg_gen_bswap16_i64(swap
, val
, 0);
3069 tcg_gen_bswap32_i64(swap
, val
, 0);
3072 tcg_gen_bswap64_i64(swap
, val
);
3075 g_assert_not_reached();
3081 addr
= plugin_prep_mem_callbacks(addr
);
3082 gen_ldst_i64(INDEX_op_qemu_st_i64
, val
, addr
, memop
, idx
);
3083 plugin_gen_mem_callbacks(addr
, oi
, QEMU_PLUGIN_MEM_W
);
3086 tcg_temp_free_i64(swap
);
3090 static void tcg_gen_ext_i32(TCGv_i32 ret
, TCGv_i32 val
, MemOp opc
)
3092 switch (opc
& MO_SSIZE
) {
3094 tcg_gen_ext8s_i32(ret
, val
);
3097 tcg_gen_ext8u_i32(ret
, val
);
3100 tcg_gen_ext16s_i32(ret
, val
);
3103 tcg_gen_ext16u_i32(ret
, val
);
3106 tcg_gen_mov_i32(ret
, val
);
3111 static void tcg_gen_ext_i64(TCGv_i64 ret
, TCGv_i64 val
, MemOp opc
)
3113 switch (opc
& MO_SSIZE
) {
3115 tcg_gen_ext8s_i64(ret
, val
);
3118 tcg_gen_ext8u_i64(ret
, val
);
3121 tcg_gen_ext16s_i64(ret
, val
);
3124 tcg_gen_ext16u_i64(ret
, val
);
3127 tcg_gen_ext32s_i64(ret
, val
);
3130 tcg_gen_ext32u_i64(ret
, val
);
3133 tcg_gen_mov_i64(ret
, val
);
3138 typedef void (*gen_atomic_cx_i32
)(TCGv_i32
, TCGv_env
, TCGv
,
3139 TCGv_i32
, TCGv_i32
, TCGv_i32
);
3140 typedef void (*gen_atomic_cx_i64
)(TCGv_i64
, TCGv_env
, TCGv
,
3141 TCGv_i64
, TCGv_i64
, TCGv_i32
);
3142 typedef void (*gen_atomic_op_i32
)(TCGv_i32
, TCGv_env
, TCGv
,
3143 TCGv_i32
, TCGv_i32
);
3144 typedef void (*gen_atomic_op_i64
)(TCGv_i64
, TCGv_env
, TCGv
,
3145 TCGv_i64
, TCGv_i32
);
3147 #ifdef CONFIG_ATOMIC64
3148 # define WITH_ATOMIC64(X) X,
3150 # define WITH_ATOMIC64(X)
3153 static void * const table_cmpxchg
[(MO_SIZE
| MO_BSWAP
) + 1] = {
3154 [MO_8
] = gen_helper_atomic_cmpxchgb
,
3155 [MO_16
| MO_LE
] = gen_helper_atomic_cmpxchgw_le
,
3156 [MO_16
| MO_BE
] = gen_helper_atomic_cmpxchgw_be
,
3157 [MO_32
| MO_LE
] = gen_helper_atomic_cmpxchgl_le
,
3158 [MO_32
| MO_BE
] = gen_helper_atomic_cmpxchgl_be
,
3159 WITH_ATOMIC64([MO_64
| MO_LE
] = gen_helper_atomic_cmpxchgq_le
)
3160 WITH_ATOMIC64([MO_64
| MO_BE
] = gen_helper_atomic_cmpxchgq_be
)
3163 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv
, TCGv addr
, TCGv_i32 cmpv
,
3164 TCGv_i32 newv
, TCGArg idx
, MemOp memop
)
3166 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3168 if (!(tcg_ctx
->tb_cflags
& CF_PARALLEL
)) {
3169 TCGv_i32 t1
= tcg_temp_new_i32();
3170 TCGv_i32 t2
= tcg_temp_new_i32();
3172 tcg_gen_ext_i32(t2
, cmpv
, memop
& MO_SIZE
);
3174 tcg_gen_qemu_ld_i32(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3175 tcg_gen_movcond_i32(TCG_COND_EQ
, t2
, t1
, t2
, newv
, t1
);
3176 tcg_gen_qemu_st_i32(t2
, addr
, idx
, memop
);
3177 tcg_temp_free_i32(t2
);
3179 if (memop
& MO_SIGN
) {
3180 tcg_gen_ext_i32(retv
, t1
, memop
);
3182 tcg_gen_mov_i32(retv
, t1
);
3184 tcg_temp_free_i32(t1
);
3186 gen_atomic_cx_i32 gen
;
3189 gen
= table_cmpxchg
[memop
& (MO_SIZE
| MO_BSWAP
)];
3190 tcg_debug_assert(gen
!= NULL
);
3192 oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3193 gen(retv
, cpu_env
, addr
, cmpv
, newv
, tcg_constant_i32(oi
));
3195 if (memop
& MO_SIGN
) {
3196 tcg_gen_ext_i32(retv
, retv
, memop
);
3201 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv
, TCGv addr
, TCGv_i64 cmpv
,
3202 TCGv_i64 newv
, TCGArg idx
, MemOp memop
)
3204 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3206 if (!(tcg_ctx
->tb_cflags
& CF_PARALLEL
)) {
3207 TCGv_i64 t1
= tcg_temp_new_i64();
3208 TCGv_i64 t2
= tcg_temp_new_i64();
3210 tcg_gen_ext_i64(t2
, cmpv
, memop
& MO_SIZE
);
3212 tcg_gen_qemu_ld_i64(t1
, addr
, idx
, memop
& ~MO_SIGN
);
3213 tcg_gen_movcond_i64(TCG_COND_EQ
, t2
, t1
, t2
, newv
, t1
);
3214 tcg_gen_qemu_st_i64(t2
, addr
, idx
, memop
);
3215 tcg_temp_free_i64(t2
);
3217 if (memop
& MO_SIGN
) {
3218 tcg_gen_ext_i64(retv
, t1
, memop
);
3220 tcg_gen_mov_i64(retv
, t1
);
3222 tcg_temp_free_i64(t1
);
3223 } else if ((memop
& MO_SIZE
) == MO_64
) {
3224 #ifdef CONFIG_ATOMIC64
3225 gen_atomic_cx_i64 gen
;
3228 gen
= table_cmpxchg
[memop
& (MO_SIZE
| MO_BSWAP
)];
3229 tcg_debug_assert(gen
!= NULL
);
3231 oi
= make_memop_idx(memop
, idx
);
3232 gen(retv
, cpu_env
, addr
, cmpv
, newv
, tcg_constant_i32(oi
));
3234 gen_helper_exit_atomic(cpu_env
);
3235 /* Produce a result, so that we have a well-formed opcode stream
3236 with respect to uses of the result in the (dead) code following. */
3237 tcg_gen_movi_i64(retv
, 0);
3238 #endif /* CONFIG_ATOMIC64 */
3240 TCGv_i32 c32
= tcg_temp_new_i32();
3241 TCGv_i32 n32
= tcg_temp_new_i32();
3242 TCGv_i32 r32
= tcg_temp_new_i32();
3244 tcg_gen_extrl_i64_i32(c32
, cmpv
);
3245 tcg_gen_extrl_i64_i32(n32
, newv
);
3246 tcg_gen_atomic_cmpxchg_i32(r32
, addr
, c32
, n32
, idx
, memop
& ~MO_SIGN
);
3247 tcg_temp_free_i32(c32
);
3248 tcg_temp_free_i32(n32
);
3250 tcg_gen_extu_i32_i64(retv
, r32
);
3251 tcg_temp_free_i32(r32
);
3253 if (memop
& MO_SIGN
) {
3254 tcg_gen_ext_i64(retv
, retv
, memop
);
3259 static void do_nonatomic_op_i32(TCGv_i32 ret
, TCGv addr
, TCGv_i32 val
,
3260 TCGArg idx
, MemOp memop
, bool new_val
,
3261 void (*gen
)(TCGv_i32
, TCGv_i32
, TCGv_i32
))
3263 TCGv_i32 t1
= tcg_temp_new_i32();
3264 TCGv_i32 t2
= tcg_temp_new_i32();
3266 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3268 tcg_gen_qemu_ld_i32(t1
, addr
, idx
, memop
);
3269 tcg_gen_ext_i32(t2
, val
, memop
);
3271 tcg_gen_qemu_st_i32(t2
, addr
, idx
, memop
);
3273 tcg_gen_ext_i32(ret
, (new_val
? t2
: t1
), memop
);
3274 tcg_temp_free_i32(t1
);
3275 tcg_temp_free_i32(t2
);
3278 static void do_atomic_op_i32(TCGv_i32 ret
, TCGv addr
, TCGv_i32 val
,
3279 TCGArg idx
, MemOp memop
, void * const table
[])
3281 gen_atomic_op_i32 gen
;
3284 memop
= tcg_canonicalize_memop(memop
, 0, 0);
3286 gen
= table
[memop
& (MO_SIZE
| MO_BSWAP
)];
3287 tcg_debug_assert(gen
!= NULL
);
3289 oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3290 gen(ret
, cpu_env
, addr
, val
, tcg_constant_i32(oi
));
3292 if (memop
& MO_SIGN
) {
3293 tcg_gen_ext_i32(ret
, ret
, memop
);
3297 static void do_nonatomic_op_i64(TCGv_i64 ret
, TCGv addr
, TCGv_i64 val
,
3298 TCGArg idx
, MemOp memop
, bool new_val
,
3299 void (*gen
)(TCGv_i64
, TCGv_i64
, TCGv_i64
))
3301 TCGv_i64 t1
= tcg_temp_new_i64();
3302 TCGv_i64 t2
= tcg_temp_new_i64();
3304 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3306 tcg_gen_qemu_ld_i64(t1
, addr
, idx
, memop
);
3307 tcg_gen_ext_i64(t2
, val
, memop
);
3309 tcg_gen_qemu_st_i64(t2
, addr
, idx
, memop
);
3311 tcg_gen_ext_i64(ret
, (new_val
? t2
: t1
), memop
);
3312 tcg_temp_free_i64(t1
);
3313 tcg_temp_free_i64(t2
);
3316 static void do_atomic_op_i64(TCGv_i64 ret
, TCGv addr
, TCGv_i64 val
,
3317 TCGArg idx
, MemOp memop
, void * const table
[])
3319 memop
= tcg_canonicalize_memop(memop
, 1, 0);
3321 if ((memop
& MO_SIZE
) == MO_64
) {
3322 #ifdef CONFIG_ATOMIC64
3323 gen_atomic_op_i64 gen
;
3326 gen
= table
[memop
& (MO_SIZE
| MO_BSWAP
)];
3327 tcg_debug_assert(gen
!= NULL
);
3329 oi
= make_memop_idx(memop
& ~MO_SIGN
, idx
);
3330 gen(ret
, cpu_env
, addr
, val
, tcg_constant_i32(oi
));
3332 gen_helper_exit_atomic(cpu_env
);
3333 /* Produce a result, so that we have a well-formed opcode stream
3334 with respect to uses of the result in the (dead) code following. */
3335 tcg_gen_movi_i64(ret
, 0);
3336 #endif /* CONFIG_ATOMIC64 */
3338 TCGv_i32 v32
= tcg_temp_new_i32();
3339 TCGv_i32 r32
= tcg_temp_new_i32();
3341 tcg_gen_extrl_i64_i32(v32
, val
);
3342 do_atomic_op_i32(r32
, addr
, v32
, idx
, memop
& ~MO_SIGN
, table
);
3343 tcg_temp_free_i32(v32
);
3345 tcg_gen_extu_i32_i64(ret
, r32
);
3346 tcg_temp_free_i32(r32
);
3348 if (memop
& MO_SIGN
) {
3349 tcg_gen_ext_i64(ret
, ret
, memop
);
3354 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3355 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
3356 [MO_8] = gen_helper_atomic_##NAME##b, \
3357 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3358 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3359 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3360 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
3361 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3362 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
3364 void tcg_gen_atomic_##NAME##_i32 \
3365 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \
3367 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3368 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3370 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3371 tcg_gen_##OP##_i32); \
3374 void tcg_gen_atomic_##NAME##_i64 \
3375 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \
3377 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3378 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3380 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3381 tcg_gen_##OP##_i64); \
3385 GEN_ATOMIC_HELPER(fetch_add
, add
, 0)
3386 GEN_ATOMIC_HELPER(fetch_and
, and, 0)
3387 GEN_ATOMIC_HELPER(fetch_or
, or, 0)
3388 GEN_ATOMIC_HELPER(fetch_xor
, xor, 0)
3389 GEN_ATOMIC_HELPER(fetch_smin
, smin
, 0)
3390 GEN_ATOMIC_HELPER(fetch_umin
, umin
, 0)
3391 GEN_ATOMIC_HELPER(fetch_smax
, smax
, 0)
3392 GEN_ATOMIC_HELPER(fetch_umax
, umax
, 0)
3394 GEN_ATOMIC_HELPER(add_fetch
, add
, 1)
3395 GEN_ATOMIC_HELPER(and_fetch
, and, 1)
3396 GEN_ATOMIC_HELPER(or_fetch
, or, 1)
3397 GEN_ATOMIC_HELPER(xor_fetch
, xor, 1)
3398 GEN_ATOMIC_HELPER(smin_fetch
, smin
, 1)
3399 GEN_ATOMIC_HELPER(umin_fetch
, umin
, 1)
3400 GEN_ATOMIC_HELPER(smax_fetch
, smax
, 1)
3401 GEN_ATOMIC_HELPER(umax_fetch
, umax
, 1)
3403 static void tcg_gen_mov2_i32(TCGv_i32 r
, TCGv_i32 a
, TCGv_i32 b
)
3405 tcg_gen_mov_i32(r
, b
);
3408 static void tcg_gen_mov2_i64(TCGv_i64 r
, TCGv_i64 a
, TCGv_i64 b
)
3410 tcg_gen_mov_i64(r
, b
);
3413 GEN_ATOMIC_HELPER(xchg
, mov2
, 0)
3415 #undef GEN_ATOMIC_HELPER