Merge remote-tracking branch 'remotes/armbru/tags/pull-build-2019-07-02-v2' into...
[qemu/ar7.git] / tcg / tcg-op.c
blob587d0922384628e449aaf85322dd4abfb8554111
1 /*
2  * Tiny Code Generator for QEMU
3  *
4  * Copyright (c) 2008 Fabrice Bellard
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
25 #include "qemu/osdep.h"
26 #include "cpu.h"
27 #include "exec/exec-all.h"
28 #include "tcg.h"
29 #include "tcg-op.h"
30 #include "tcg-mo.h"
31 #include "trace-tcg.h"
32 #include "trace/mem.h"
34 /* Reduce the number of ifdefs below.  This assumes that all uses of
35    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
36    the compiler can eliminate.  */
37 #if TCG_TARGET_REG_BITS == 64
38 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
39 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
40 #define TCGV_LOW  TCGV_LOW_link_error
41 #define TCGV_HIGH TCGV_HIGH_link_error
42 #endif
44 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
46     TCGOp *op = tcg_emit_op(opc);
47     op->args[0] = a1;
50 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
52     TCGOp *op = tcg_emit_op(opc);
53     op->args[0] = a1;
54     op->args[1] = a2;
57 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
59     TCGOp *op = tcg_emit_op(opc);
60     op->args[0] = a1;
61     op->args[1] = a2;
62     op->args[2] = a3;
65 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
67     TCGOp *op = tcg_emit_op(opc);
68     op->args[0] = a1;
69     op->args[1] = a2;
70     op->args[2] = a3;
71     op->args[3] = a4;
74 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
75                  TCGArg a4, TCGArg a5)
77     TCGOp *op = tcg_emit_op(opc);
78     op->args[0] = a1;
79     op->args[1] = a2;
80     op->args[2] = a3;
81     op->args[3] = a4;
82     op->args[4] = a5;
85 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
86                  TCGArg a4, TCGArg a5, TCGArg a6)
88     TCGOp *op = tcg_emit_op(opc);
89     op->args[0] = a1;
90     op->args[1] = a2;
91     op->args[2] = a3;
92     op->args[3] = a4;
93     op->args[4] = a5;
94     op->args[5] = a6;
97 void tcg_gen_mb(TCGBar mb_type)
99     if (tcg_ctx->tb_cflags & CF_PARALLEL) {
100         tcg_gen_op1(INDEX_op_mb, mb_type);
101     }
104 /* 32 bit ops */
106 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
108     /* some cases can be optimized here */
109     if (arg2 == 0) {
110         tcg_gen_mov_i32(ret, arg1);
111     } else {
112         TCGv_i32 t0 = tcg_const_i32(arg2);
113         tcg_gen_add_i32(ret, arg1, t0);
114         tcg_temp_free_i32(t0);
115     }
118 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120     if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
121         /* Don't recurse with tcg_gen_neg_i32.  */
122         tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
123     } else {
124         TCGv_i32 t0 = tcg_const_i32(arg1);
125         tcg_gen_sub_i32(ret, t0, arg2);
126         tcg_temp_free_i32(t0);
127     }
130 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
132     /* some cases can be optimized here */
133     if (arg2 == 0) {
134         tcg_gen_mov_i32(ret, arg1);
135     } else {
136         TCGv_i32 t0 = tcg_const_i32(arg2);
137         tcg_gen_sub_i32(ret, arg1, t0);
138         tcg_temp_free_i32(t0);
139     }
142 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
144     TCGv_i32 t0;
145     /* Some cases can be optimized here.  */
146     switch (arg2) {
147     case 0:
148         tcg_gen_movi_i32(ret, 0);
149         return;
150     case -1:
151         tcg_gen_mov_i32(ret, arg1);
152         return;
153     case 0xff:
154         /* Don't recurse with tcg_gen_ext8u_i32.  */
155         if (TCG_TARGET_HAS_ext8u_i32) {
156             tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
157             return;
158         }
159         break;
160     case 0xffff:
161         if (TCG_TARGET_HAS_ext16u_i32) {
162             tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
163             return;
164         }
165         break;
166     }
167     t0 = tcg_const_i32(arg2);
168     tcg_gen_and_i32(ret, arg1, t0);
169     tcg_temp_free_i32(t0);
172 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
174     /* Some cases can be optimized here.  */
175     if (arg2 == -1) {
176         tcg_gen_movi_i32(ret, -1);
177     } else if (arg2 == 0) {
178         tcg_gen_mov_i32(ret, arg1);
179     } else {
180         TCGv_i32 t0 = tcg_const_i32(arg2);
181         tcg_gen_or_i32(ret, arg1, t0);
182         tcg_temp_free_i32(t0);
183     }
186 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
188     /* Some cases can be optimized here.  */
189     if (arg2 == 0) {
190         tcg_gen_mov_i32(ret, arg1);
191     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
192         /* Don't recurse with tcg_gen_not_i32.  */
193         tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
194     } else {
195         TCGv_i32 t0 = tcg_const_i32(arg2);
196         tcg_gen_xor_i32(ret, arg1, t0);
197         tcg_temp_free_i32(t0);
198     }
201 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
203     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
204     if (arg2 == 0) {
205         tcg_gen_mov_i32(ret, arg1);
206     } else {
207         TCGv_i32 t0 = tcg_const_i32(arg2);
208         tcg_gen_shl_i32(ret, arg1, t0);
209         tcg_temp_free_i32(t0);
210     }
213 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
215     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
216     if (arg2 == 0) {
217         tcg_gen_mov_i32(ret, arg1);
218     } else {
219         TCGv_i32 t0 = tcg_const_i32(arg2);
220         tcg_gen_shr_i32(ret, arg1, t0);
221         tcg_temp_free_i32(t0);
222     }
225 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
227     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
228     if (arg2 == 0) {
229         tcg_gen_mov_i32(ret, arg1);
230     } else {
231         TCGv_i32 t0 = tcg_const_i32(arg2);
232         tcg_gen_sar_i32(ret, arg1, t0);
233         tcg_temp_free_i32(t0);
234     }
237 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
239     if (cond == TCG_COND_ALWAYS) {
240         tcg_gen_br(l);
241     } else if (cond != TCG_COND_NEVER) {
242         l->refs++;
243         tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
244     }
247 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
249     if (cond == TCG_COND_ALWAYS) {
250         tcg_gen_br(l);
251     } else if (cond != TCG_COND_NEVER) {
252         TCGv_i32 t0 = tcg_const_i32(arg2);
253         tcg_gen_brcond_i32(cond, arg1, t0, l);
254         tcg_temp_free_i32(t0);
255     }
258 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
259                          TCGv_i32 arg1, TCGv_i32 arg2)
261     if (cond == TCG_COND_ALWAYS) {
262         tcg_gen_movi_i32(ret, 1);
263     } else if (cond == TCG_COND_NEVER) {
264         tcg_gen_movi_i32(ret, 0);
265     } else {
266         tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
267     }
270 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
271                           TCGv_i32 arg1, int32_t arg2)
273     TCGv_i32 t0 = tcg_const_i32(arg2);
274     tcg_gen_setcond_i32(cond, ret, arg1, t0);
275     tcg_temp_free_i32(t0);
278 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
280     if (arg2 == 0) {
281         tcg_gen_movi_i32(ret, 0);
282     } else if (is_power_of_2(arg2)) {
283         tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
284     } else {
285         TCGv_i32 t0 = tcg_const_i32(arg2);
286         tcg_gen_mul_i32(ret, arg1, t0);
287         tcg_temp_free_i32(t0);
288     }
291 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
293     if (TCG_TARGET_HAS_div_i32) {
294         tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
295     } else if (TCG_TARGET_HAS_div2_i32) {
296         TCGv_i32 t0 = tcg_temp_new_i32();
297         tcg_gen_sari_i32(t0, arg1, 31);
298         tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
299         tcg_temp_free_i32(t0);
300     } else {
301         gen_helper_div_i32(ret, arg1, arg2);
302     }
305 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
307     if (TCG_TARGET_HAS_rem_i32) {
308         tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
309     } else if (TCG_TARGET_HAS_div_i32) {
310         TCGv_i32 t0 = tcg_temp_new_i32();
311         tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
312         tcg_gen_mul_i32(t0, t0, arg2);
313         tcg_gen_sub_i32(ret, arg1, t0);
314         tcg_temp_free_i32(t0);
315     } else if (TCG_TARGET_HAS_div2_i32) {
316         TCGv_i32 t0 = tcg_temp_new_i32();
317         tcg_gen_sari_i32(t0, arg1, 31);
318         tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
319         tcg_temp_free_i32(t0);
320     } else {
321         gen_helper_rem_i32(ret, arg1, arg2);
322     }
325 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
327     if (TCG_TARGET_HAS_div_i32) {
328         tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
329     } else if (TCG_TARGET_HAS_div2_i32) {
330         TCGv_i32 t0 = tcg_temp_new_i32();
331         tcg_gen_movi_i32(t0, 0);
332         tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
333         tcg_temp_free_i32(t0);
334     } else {
335         gen_helper_divu_i32(ret, arg1, arg2);
336     }
339 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
341     if (TCG_TARGET_HAS_rem_i32) {
342         tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
343     } else if (TCG_TARGET_HAS_div_i32) {
344         TCGv_i32 t0 = tcg_temp_new_i32();
345         tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
346         tcg_gen_mul_i32(t0, t0, arg2);
347         tcg_gen_sub_i32(ret, arg1, t0);
348         tcg_temp_free_i32(t0);
349     } else if (TCG_TARGET_HAS_div2_i32) {
350         TCGv_i32 t0 = tcg_temp_new_i32();
351         tcg_gen_movi_i32(t0, 0);
352         tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
353         tcg_temp_free_i32(t0);
354     } else {
355         gen_helper_remu_i32(ret, arg1, arg2);
356     }
359 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
361     if (TCG_TARGET_HAS_andc_i32) {
362         tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
363     } else {
364         TCGv_i32 t0 = tcg_temp_new_i32();
365         tcg_gen_not_i32(t0, arg2);
366         tcg_gen_and_i32(ret, arg1, t0);
367         tcg_temp_free_i32(t0);
368     }
371 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
373     if (TCG_TARGET_HAS_eqv_i32) {
374         tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
375     } else {
376         tcg_gen_xor_i32(ret, arg1, arg2);
377         tcg_gen_not_i32(ret, ret);
378     }
381 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
383     if (TCG_TARGET_HAS_nand_i32) {
384         tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
385     } else {
386         tcg_gen_and_i32(ret, arg1, arg2);
387         tcg_gen_not_i32(ret, ret);
388     }
391 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
393     if (TCG_TARGET_HAS_nor_i32) {
394         tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
395     } else {
396         tcg_gen_or_i32(ret, arg1, arg2);
397         tcg_gen_not_i32(ret, ret);
398     }
401 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
403     if (TCG_TARGET_HAS_orc_i32) {
404         tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
405     } else {
406         TCGv_i32 t0 = tcg_temp_new_i32();
407         tcg_gen_not_i32(t0, arg2);
408         tcg_gen_or_i32(ret, arg1, t0);
409         tcg_temp_free_i32(t0);
410     }
413 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
415     if (TCG_TARGET_HAS_clz_i32) {
416         tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
417     } else if (TCG_TARGET_HAS_clz_i64) {
418         TCGv_i64 t1 = tcg_temp_new_i64();
419         TCGv_i64 t2 = tcg_temp_new_i64();
420         tcg_gen_extu_i32_i64(t1, arg1);
421         tcg_gen_extu_i32_i64(t2, arg2);
422         tcg_gen_addi_i64(t2, t2, 32);
423         tcg_gen_clz_i64(t1, t1, t2);
424         tcg_gen_extrl_i64_i32(ret, t1);
425         tcg_temp_free_i64(t1);
426         tcg_temp_free_i64(t2);
427         tcg_gen_subi_i32(ret, ret, 32);
428     } else {
429         gen_helper_clz_i32(ret, arg1, arg2);
430     }
433 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
435     TCGv_i32 t = tcg_const_i32(arg2);
436     tcg_gen_clz_i32(ret, arg1, t);
437     tcg_temp_free_i32(t);
440 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
442     if (TCG_TARGET_HAS_ctz_i32) {
443         tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
444     } else if (TCG_TARGET_HAS_ctz_i64) {
445         TCGv_i64 t1 = tcg_temp_new_i64();
446         TCGv_i64 t2 = tcg_temp_new_i64();
447         tcg_gen_extu_i32_i64(t1, arg1);
448         tcg_gen_extu_i32_i64(t2, arg2);
449         tcg_gen_ctz_i64(t1, t1, t2);
450         tcg_gen_extrl_i64_i32(ret, t1);
451         tcg_temp_free_i64(t1);
452         tcg_temp_free_i64(t2);
453     } else if (TCG_TARGET_HAS_ctpop_i32
454                || TCG_TARGET_HAS_ctpop_i64
455                || TCG_TARGET_HAS_clz_i32
456                || TCG_TARGET_HAS_clz_i64) {
457         TCGv_i32 z, t = tcg_temp_new_i32();
459         if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
460             tcg_gen_subi_i32(t, arg1, 1);
461             tcg_gen_andc_i32(t, t, arg1);
462             tcg_gen_ctpop_i32(t, t);
463         } else {
464             /* Since all non-x86 hosts have clz(0) == 32, don't fight it.  */
465             tcg_gen_neg_i32(t, arg1);
466             tcg_gen_and_i32(t, t, arg1);
467             tcg_gen_clzi_i32(t, t, 32);
468             tcg_gen_xori_i32(t, t, 31);
469         }
470         z = tcg_const_i32(0);
471         tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
472         tcg_temp_free_i32(t);
473         tcg_temp_free_i32(z);
474     } else {
475         gen_helper_ctz_i32(ret, arg1, arg2);
476     }
479 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
481     if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
482         /* This equivalence has the advantage of not requiring a fixup.  */
483         TCGv_i32 t = tcg_temp_new_i32();
484         tcg_gen_subi_i32(t, arg1, 1);
485         tcg_gen_andc_i32(t, t, arg1);
486         tcg_gen_ctpop_i32(ret, t);
487         tcg_temp_free_i32(t);
488     } else {
489         TCGv_i32 t = tcg_const_i32(arg2);
490         tcg_gen_ctz_i32(ret, arg1, t);
491         tcg_temp_free_i32(t);
492     }
495 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
497     if (TCG_TARGET_HAS_clz_i32) {
498         TCGv_i32 t = tcg_temp_new_i32();
499         tcg_gen_sari_i32(t, arg, 31);
500         tcg_gen_xor_i32(t, t, arg);
501         tcg_gen_clzi_i32(t, t, 32);
502         tcg_gen_subi_i32(ret, t, 1);
503         tcg_temp_free_i32(t);
504     } else {
505         gen_helper_clrsb_i32(ret, arg);
506     }
509 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
511     if (TCG_TARGET_HAS_ctpop_i32) {
512         tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
513     } else if (TCG_TARGET_HAS_ctpop_i64) {
514         TCGv_i64 t = tcg_temp_new_i64();
515         tcg_gen_extu_i32_i64(t, arg1);
516         tcg_gen_ctpop_i64(t, t);
517         tcg_gen_extrl_i64_i32(ret, t);
518         tcg_temp_free_i64(t);
519     } else {
520         gen_helper_ctpop_i32(ret, arg1);
521     }
524 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
526     if (TCG_TARGET_HAS_rot_i32) {
527         tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
528     } else {
529         TCGv_i32 t0, t1;
531         t0 = tcg_temp_new_i32();
532         t1 = tcg_temp_new_i32();
533         tcg_gen_shl_i32(t0, arg1, arg2);
534         tcg_gen_subfi_i32(t1, 32, arg2);
535         tcg_gen_shr_i32(t1, arg1, t1);
536         tcg_gen_or_i32(ret, t0, t1);
537         tcg_temp_free_i32(t0);
538         tcg_temp_free_i32(t1);
539     }
542 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
544     tcg_debug_assert(arg2 < 32);
545     /* some cases can be optimized here */
546     if (arg2 == 0) {
547         tcg_gen_mov_i32(ret, arg1);
548     } else if (TCG_TARGET_HAS_rot_i32) {
549         TCGv_i32 t0 = tcg_const_i32(arg2);
550         tcg_gen_rotl_i32(ret, arg1, t0);
551         tcg_temp_free_i32(t0);
552     } else {
553         TCGv_i32 t0, t1;
554         t0 = tcg_temp_new_i32();
555         t1 = tcg_temp_new_i32();
556         tcg_gen_shli_i32(t0, arg1, arg2);
557         tcg_gen_shri_i32(t1, arg1, 32 - arg2);
558         tcg_gen_or_i32(ret, t0, t1);
559         tcg_temp_free_i32(t0);
560         tcg_temp_free_i32(t1);
561     }
564 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
566     if (TCG_TARGET_HAS_rot_i32) {
567         tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
568     } else {
569         TCGv_i32 t0, t1;
571         t0 = tcg_temp_new_i32();
572         t1 = tcg_temp_new_i32();
573         tcg_gen_shr_i32(t0, arg1, arg2);
574         tcg_gen_subfi_i32(t1, 32, arg2);
575         tcg_gen_shl_i32(t1, arg1, t1);
576         tcg_gen_or_i32(ret, t0, t1);
577         tcg_temp_free_i32(t0);
578         tcg_temp_free_i32(t1);
579     }
582 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
584     tcg_debug_assert(arg2 < 32);
585     /* some cases can be optimized here */
586     if (arg2 == 0) {
587         tcg_gen_mov_i32(ret, arg1);
588     } else {
589         tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
590     }
593 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
594                          unsigned int ofs, unsigned int len)
596     uint32_t mask;
597     TCGv_i32 t1;
599     tcg_debug_assert(ofs < 32);
600     tcg_debug_assert(len > 0);
601     tcg_debug_assert(len <= 32);
602     tcg_debug_assert(ofs + len <= 32);
604     if (len == 32) {
605         tcg_gen_mov_i32(ret, arg2);
606         return;
607     }
608     if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
609         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
610         return;
611     }
613     t1 = tcg_temp_new_i32();
615     if (TCG_TARGET_HAS_extract2_i32) {
616         if (ofs + len == 32) {
617             tcg_gen_shli_i32(t1, arg1, len);
618             tcg_gen_extract2_i32(ret, t1, arg2, len);
619             goto done;
620         }
621         if (ofs == 0) {
622             tcg_gen_extract2_i32(ret, arg1, arg2, len);
623             tcg_gen_rotli_i32(ret, ret, len);
624             goto done;
625         }
626     }
628     mask = (1u << len) - 1;
629     if (ofs + len < 32) {
630         tcg_gen_andi_i32(t1, arg2, mask);
631         tcg_gen_shli_i32(t1, t1, ofs);
632     } else {
633         tcg_gen_shli_i32(t1, arg2, ofs);
634     }
635     tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
636     tcg_gen_or_i32(ret, ret, t1);
637  done:
638     tcg_temp_free_i32(t1);
641 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
642                            unsigned int ofs, unsigned int len)
644     tcg_debug_assert(ofs < 32);
645     tcg_debug_assert(len > 0);
646     tcg_debug_assert(len <= 32);
647     tcg_debug_assert(ofs + len <= 32);
649     if (ofs + len == 32) {
650         tcg_gen_shli_i32(ret, arg, ofs);
651     } else if (ofs == 0) {
652         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
653     } else if (TCG_TARGET_HAS_deposit_i32
654                && TCG_TARGET_deposit_i32_valid(ofs, len)) {
655         TCGv_i32 zero = tcg_const_i32(0);
656         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
657         tcg_temp_free_i32(zero);
658     } else {
659         /* To help two-operand hosts we prefer to zero-extend first,
660            which allows ARG to stay live.  */
661         switch (len) {
662         case 16:
663             if (TCG_TARGET_HAS_ext16u_i32) {
664                 tcg_gen_ext16u_i32(ret, arg);
665                 tcg_gen_shli_i32(ret, ret, ofs);
666                 return;
667             }
668             break;
669         case 8:
670             if (TCG_TARGET_HAS_ext8u_i32) {
671                 tcg_gen_ext8u_i32(ret, arg);
672                 tcg_gen_shli_i32(ret, ret, ofs);
673                 return;
674             }
675             break;
676         }
677         /* Otherwise prefer zero-extension over AND for code size.  */
678         switch (ofs + len) {
679         case 16:
680             if (TCG_TARGET_HAS_ext16u_i32) {
681                 tcg_gen_shli_i32(ret, arg, ofs);
682                 tcg_gen_ext16u_i32(ret, ret);
683                 return;
684             }
685             break;
686         case 8:
687             if (TCG_TARGET_HAS_ext8u_i32) {
688                 tcg_gen_shli_i32(ret, arg, ofs);
689                 tcg_gen_ext8u_i32(ret, ret);
690                 return;
691             }
692             break;
693         }
694         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
695         tcg_gen_shli_i32(ret, ret, ofs);
696     }
699 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
700                          unsigned int ofs, unsigned int len)
702     tcg_debug_assert(ofs < 32);
703     tcg_debug_assert(len > 0);
704     tcg_debug_assert(len <= 32);
705     tcg_debug_assert(ofs + len <= 32);
707     /* Canonicalize certain special cases, even if extract is supported.  */
708     if (ofs + len == 32) {
709         tcg_gen_shri_i32(ret, arg, 32 - len);
710         return;
711     }
712     if (ofs == 0) {
713         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
714         return;
715     }
717     if (TCG_TARGET_HAS_extract_i32
718         && TCG_TARGET_extract_i32_valid(ofs, len)) {
719         tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
720         return;
721     }
723     /* Assume that zero-extension, if available, is cheaper than a shift.  */
724     switch (ofs + len) {
725     case 16:
726         if (TCG_TARGET_HAS_ext16u_i32) {
727             tcg_gen_ext16u_i32(ret, arg);
728             tcg_gen_shri_i32(ret, ret, ofs);
729             return;
730         }
731         break;
732     case 8:
733         if (TCG_TARGET_HAS_ext8u_i32) {
734             tcg_gen_ext8u_i32(ret, arg);
735             tcg_gen_shri_i32(ret, ret, ofs);
736             return;
737         }
738         break;
739     }
741     /* ??? Ideally we'd know what values are available for immediate AND.
742        Assume that 8 bits are available, plus the special case of 16,
743        so that we get ext8u, ext16u.  */
744     switch (len) {
745     case 1 ... 8: case 16:
746         tcg_gen_shri_i32(ret, arg, ofs);
747         tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
748         break;
749     default:
750         tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
751         tcg_gen_shri_i32(ret, ret, 32 - len);
752         break;
753     }
756 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
757                           unsigned int ofs, unsigned int len)
759     tcg_debug_assert(ofs < 32);
760     tcg_debug_assert(len > 0);
761     tcg_debug_assert(len <= 32);
762     tcg_debug_assert(ofs + len <= 32);
764     /* Canonicalize certain special cases, even if extract is supported.  */
765     if (ofs + len == 32) {
766         tcg_gen_sari_i32(ret, arg, 32 - len);
767         return;
768     }
769     if (ofs == 0) {
770         switch (len) {
771         case 16:
772             tcg_gen_ext16s_i32(ret, arg);
773             return;
774         case 8:
775             tcg_gen_ext8s_i32(ret, arg);
776             return;
777         }
778     }
780     if (TCG_TARGET_HAS_sextract_i32
781         && TCG_TARGET_extract_i32_valid(ofs, len)) {
782         tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
783         return;
784     }
786     /* Assume that sign-extension, if available, is cheaper than a shift.  */
787     switch (ofs + len) {
788     case 16:
789         if (TCG_TARGET_HAS_ext16s_i32) {
790             tcg_gen_ext16s_i32(ret, arg);
791             tcg_gen_sari_i32(ret, ret, ofs);
792             return;
793         }
794         break;
795     case 8:
796         if (TCG_TARGET_HAS_ext8s_i32) {
797             tcg_gen_ext8s_i32(ret, arg);
798             tcg_gen_sari_i32(ret, ret, ofs);
799             return;
800         }
801         break;
802     }
803     switch (len) {
804     case 16:
805         if (TCG_TARGET_HAS_ext16s_i32) {
806             tcg_gen_shri_i32(ret, arg, ofs);
807             tcg_gen_ext16s_i32(ret, ret);
808             return;
809         }
810         break;
811     case 8:
812         if (TCG_TARGET_HAS_ext8s_i32) {
813             tcg_gen_shri_i32(ret, arg, ofs);
814             tcg_gen_ext8s_i32(ret, ret);
815             return;
816         }
817         break;
818     }
820     tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
821     tcg_gen_sari_i32(ret, ret, 32 - len);
825  * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
826  * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
827  */
828 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
829                           unsigned int ofs)
831     tcg_debug_assert(ofs <= 32);
832     if (ofs == 0) {
833         tcg_gen_mov_i32(ret, al);
834     } else if (ofs == 32) {
835         tcg_gen_mov_i32(ret, ah);
836     } else if (al == ah) {
837         tcg_gen_rotri_i32(ret, al, ofs);
838     } else if (TCG_TARGET_HAS_extract2_i32) {
839         tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
840     } else {
841         TCGv_i32 t0 = tcg_temp_new_i32();
842         tcg_gen_shri_i32(t0, al, ofs);
843         tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
844         tcg_temp_free_i32(t0);
845     }
848 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
849                          TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
851     if (cond == TCG_COND_ALWAYS) {
852         tcg_gen_mov_i32(ret, v1);
853     } else if (cond == TCG_COND_NEVER) {
854         tcg_gen_mov_i32(ret, v2);
855     } else if (TCG_TARGET_HAS_movcond_i32) {
856         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
857     } else {
858         TCGv_i32 t0 = tcg_temp_new_i32();
859         TCGv_i32 t1 = tcg_temp_new_i32();
860         tcg_gen_setcond_i32(cond, t0, c1, c2);
861         tcg_gen_neg_i32(t0, t0);
862         tcg_gen_and_i32(t1, v1, t0);
863         tcg_gen_andc_i32(ret, v2, t0);
864         tcg_gen_or_i32(ret, ret, t1);
865         tcg_temp_free_i32(t0);
866         tcg_temp_free_i32(t1);
867     }
870 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
871                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
873     if (TCG_TARGET_HAS_add2_i32) {
874         tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
875     } else {
876         TCGv_i64 t0 = tcg_temp_new_i64();
877         TCGv_i64 t1 = tcg_temp_new_i64();
878         tcg_gen_concat_i32_i64(t0, al, ah);
879         tcg_gen_concat_i32_i64(t1, bl, bh);
880         tcg_gen_add_i64(t0, t0, t1);
881         tcg_gen_extr_i64_i32(rl, rh, t0);
882         tcg_temp_free_i64(t0);
883         tcg_temp_free_i64(t1);
884     }
887 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
888                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
890     if (TCG_TARGET_HAS_sub2_i32) {
891         tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
892     } else {
893         TCGv_i64 t0 = tcg_temp_new_i64();
894         TCGv_i64 t1 = tcg_temp_new_i64();
895         tcg_gen_concat_i32_i64(t0, al, ah);
896         tcg_gen_concat_i32_i64(t1, bl, bh);
897         tcg_gen_sub_i64(t0, t0, t1);
898         tcg_gen_extr_i64_i32(rl, rh, t0);
899         tcg_temp_free_i64(t0);
900         tcg_temp_free_i64(t1);
901     }
904 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
906     if (TCG_TARGET_HAS_mulu2_i32) {
907         tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
908     } else if (TCG_TARGET_HAS_muluh_i32) {
909         TCGv_i32 t = tcg_temp_new_i32();
910         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
911         tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
912         tcg_gen_mov_i32(rl, t);
913         tcg_temp_free_i32(t);
914     } else {
915         TCGv_i64 t0 = tcg_temp_new_i64();
916         TCGv_i64 t1 = tcg_temp_new_i64();
917         tcg_gen_extu_i32_i64(t0, arg1);
918         tcg_gen_extu_i32_i64(t1, arg2);
919         tcg_gen_mul_i64(t0, t0, t1);
920         tcg_gen_extr_i64_i32(rl, rh, t0);
921         tcg_temp_free_i64(t0);
922         tcg_temp_free_i64(t1);
923     }
926 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
928     if (TCG_TARGET_HAS_muls2_i32) {
929         tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
930     } else if (TCG_TARGET_HAS_mulsh_i32) {
931         TCGv_i32 t = tcg_temp_new_i32();
932         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
933         tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
934         tcg_gen_mov_i32(rl, t);
935         tcg_temp_free_i32(t);
936     } else if (TCG_TARGET_REG_BITS == 32) {
937         TCGv_i32 t0 = tcg_temp_new_i32();
938         TCGv_i32 t1 = tcg_temp_new_i32();
939         TCGv_i32 t2 = tcg_temp_new_i32();
940         TCGv_i32 t3 = tcg_temp_new_i32();
941         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
942         /* Adjust for negative inputs.  */
943         tcg_gen_sari_i32(t2, arg1, 31);
944         tcg_gen_sari_i32(t3, arg2, 31);
945         tcg_gen_and_i32(t2, t2, arg2);
946         tcg_gen_and_i32(t3, t3, arg1);
947         tcg_gen_sub_i32(rh, t1, t2);
948         tcg_gen_sub_i32(rh, rh, t3);
949         tcg_gen_mov_i32(rl, t0);
950         tcg_temp_free_i32(t0);
951         tcg_temp_free_i32(t1);
952         tcg_temp_free_i32(t2);
953         tcg_temp_free_i32(t3);
954     } else {
955         TCGv_i64 t0 = tcg_temp_new_i64();
956         TCGv_i64 t1 = tcg_temp_new_i64();
957         tcg_gen_ext_i32_i64(t0, arg1);
958         tcg_gen_ext_i32_i64(t1, arg2);
959         tcg_gen_mul_i64(t0, t0, t1);
960         tcg_gen_extr_i64_i32(rl, rh, t0);
961         tcg_temp_free_i64(t0);
962         tcg_temp_free_i64(t1);
963     }
966 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
968     if (TCG_TARGET_REG_BITS == 32) {
969         TCGv_i32 t0 = tcg_temp_new_i32();
970         TCGv_i32 t1 = tcg_temp_new_i32();
971         TCGv_i32 t2 = tcg_temp_new_i32();
972         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
973         /* Adjust for negative input for the signed arg1.  */
974         tcg_gen_sari_i32(t2, arg1, 31);
975         tcg_gen_and_i32(t2, t2, arg2);
976         tcg_gen_sub_i32(rh, t1, t2);
977         tcg_gen_mov_i32(rl, t0);
978         tcg_temp_free_i32(t0);
979         tcg_temp_free_i32(t1);
980         tcg_temp_free_i32(t2);
981     } else {
982         TCGv_i64 t0 = tcg_temp_new_i64();
983         TCGv_i64 t1 = tcg_temp_new_i64();
984         tcg_gen_ext_i32_i64(t0, arg1);
985         tcg_gen_extu_i32_i64(t1, arg2);
986         tcg_gen_mul_i64(t0, t0, t1);
987         tcg_gen_extr_i64_i32(rl, rh, t0);
988         tcg_temp_free_i64(t0);
989         tcg_temp_free_i64(t1);
990     }
993 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
995     if (TCG_TARGET_HAS_ext8s_i32) {
996         tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
997     } else {
998         tcg_gen_shli_i32(ret, arg, 24);
999         tcg_gen_sari_i32(ret, ret, 24);
1000     }
1003 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
1005     if (TCG_TARGET_HAS_ext16s_i32) {
1006         tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
1007     } else {
1008         tcg_gen_shli_i32(ret, arg, 16);
1009         tcg_gen_sari_i32(ret, ret, 16);
1010     }
1013 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
1015     if (TCG_TARGET_HAS_ext8u_i32) {
1016         tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
1017     } else {
1018         tcg_gen_andi_i32(ret, arg, 0xffu);
1019     }
1022 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
1024     if (TCG_TARGET_HAS_ext16u_i32) {
1025         tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
1026     } else {
1027         tcg_gen_andi_i32(ret, arg, 0xffffu);
1028     }
1031 /* Note: we assume the two high bytes are set to zero */
1032 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
1034     if (TCG_TARGET_HAS_bswap16_i32) {
1035         tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
1036     } else {
1037         TCGv_i32 t0 = tcg_temp_new_i32();
1039         tcg_gen_ext8u_i32(t0, arg);
1040         tcg_gen_shli_i32(t0, t0, 8);
1041         tcg_gen_shri_i32(ret, arg, 8);
1042         tcg_gen_or_i32(ret, ret, t0);
1043         tcg_temp_free_i32(t0);
1044     }
1047 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1049     if (TCG_TARGET_HAS_bswap32_i32) {
1050         tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1051     } else {
1052         TCGv_i32 t0 = tcg_temp_new_i32();
1053         TCGv_i32 t1 = tcg_temp_new_i32();
1054         TCGv_i32 t2 = tcg_const_i32(0x00ff00ff);
1056                                         /* arg = abcd */
1057         tcg_gen_shri_i32(t0, arg, 8);   /*  t0 = .abc */
1058         tcg_gen_and_i32(t1, arg, t2);   /*  t1 = .b.d */
1059         tcg_gen_and_i32(t0, t0, t2);    /*  t0 = .a.c */
1060         tcg_gen_shli_i32(t1, t1, 8);    /*  t1 = b.d. */
1061         tcg_gen_or_i32(ret, t0, t1);    /* ret = badc */
1063         tcg_gen_shri_i32(t0, ret, 16);  /*  t0 = ..ba */
1064         tcg_gen_shli_i32(t1, ret, 16);  /*  t1 = dc.. */
1065         tcg_gen_or_i32(ret, t0, t1);    /* ret = dcba */
1067         tcg_temp_free_i32(t0);
1068         tcg_temp_free_i32(t1);
1069         tcg_temp_free_i32(t2);
1070     }
1073 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1075     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1078 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1080     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1083 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1085     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1088 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1090     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1093 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1095     TCGv_i32 t = tcg_temp_new_i32();
1097     tcg_gen_sari_i32(t, a, 31);
1098     tcg_gen_xor_i32(ret, a, t);
1099     tcg_gen_sub_i32(ret, ret, t);
1100     tcg_temp_free_i32(t);
1103 /* 64-bit ops */
1105 #if TCG_TARGET_REG_BITS == 32
1106 /* These are all inline for TCG_TARGET_REG_BITS == 64.  */
1108 void tcg_gen_discard_i64(TCGv_i64 arg)
1110     tcg_gen_discard_i32(TCGV_LOW(arg));
1111     tcg_gen_discard_i32(TCGV_HIGH(arg));
1114 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1116     tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1117     tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1120 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1122     tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1123     tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1126 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1128     tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1129     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1132 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1134     tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1135     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1138 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1140     tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1141     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1144 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1146     tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1147     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1150 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1152     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1153     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1156 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1158     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1159     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1162 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1164     /* Since arg2 and ret have different types,
1165        they cannot be the same temporary */
1166 #ifdef HOST_WORDS_BIGENDIAN
1167     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1168     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1169 #else
1170     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1171     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1172 #endif
1175 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1177 #ifdef HOST_WORDS_BIGENDIAN
1178     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1179     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1180 #else
1181     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1182     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1183 #endif
1186 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1188     tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1189     tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1192 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1194     tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1195     tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1198 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1200     tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1201     tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1204 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1206     gen_helper_shl_i64(ret, arg1, arg2);
1209 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1211     gen_helper_shr_i64(ret, arg1, arg2);
1214 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1216     gen_helper_sar_i64(ret, arg1, arg2);
1219 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1221     TCGv_i64 t0;
1222     TCGv_i32 t1;
1224     t0 = tcg_temp_new_i64();
1225     t1 = tcg_temp_new_i32();
1227     tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1228                       TCGV_LOW(arg1), TCGV_LOW(arg2));
1230     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1231     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1232     tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1233     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1235     tcg_gen_mov_i64(ret, t0);
1236     tcg_temp_free_i64(t0);
1237     tcg_temp_free_i32(t1);
1239 #endif /* TCG_TARGET_REG_SIZE == 32 */
1241 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1243     /* some cases can be optimized here */
1244     if (arg2 == 0) {
1245         tcg_gen_mov_i64(ret, arg1);
1246     } else {
1247         TCGv_i64 t0 = tcg_const_i64(arg2);
1248         tcg_gen_add_i64(ret, arg1, t0);
1249         tcg_temp_free_i64(t0);
1250     }
1253 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1255     if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1256         /* Don't recurse with tcg_gen_neg_i64.  */
1257         tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1258     } else {
1259         TCGv_i64 t0 = tcg_const_i64(arg1);
1260         tcg_gen_sub_i64(ret, t0, arg2);
1261         tcg_temp_free_i64(t0);
1262     }
1265 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1267     /* some cases can be optimized here */
1268     if (arg2 == 0) {
1269         tcg_gen_mov_i64(ret, arg1);
1270     } else {
1271         TCGv_i64 t0 = tcg_const_i64(arg2);
1272         tcg_gen_sub_i64(ret, arg1, t0);
1273         tcg_temp_free_i64(t0);
1274     }
1277 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1279     TCGv_i64 t0;
1281     if (TCG_TARGET_REG_BITS == 32) {
1282         tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1283         tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1284         return;
1285     }
1287     /* Some cases can be optimized here.  */
1288     switch (arg2) {
1289     case 0:
1290         tcg_gen_movi_i64(ret, 0);
1291         return;
1292     case -1:
1293         tcg_gen_mov_i64(ret, arg1);
1294         return;
1295     case 0xff:
1296         /* Don't recurse with tcg_gen_ext8u_i64.  */
1297         if (TCG_TARGET_HAS_ext8u_i64) {
1298             tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1299             return;
1300         }
1301         break;
1302     case 0xffff:
1303         if (TCG_TARGET_HAS_ext16u_i64) {
1304             tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1305             return;
1306         }
1307         break;
1308     case 0xffffffffu:
1309         if (TCG_TARGET_HAS_ext32u_i64) {
1310             tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1311             return;
1312         }
1313         break;
1314     }
1315     t0 = tcg_const_i64(arg2);
1316     tcg_gen_and_i64(ret, arg1, t0);
1317     tcg_temp_free_i64(t0);
1320 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1322     if (TCG_TARGET_REG_BITS == 32) {
1323         tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1324         tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1325         return;
1326     }
1327     /* Some cases can be optimized here.  */
1328     if (arg2 == -1) {
1329         tcg_gen_movi_i64(ret, -1);
1330     } else if (arg2 == 0) {
1331         tcg_gen_mov_i64(ret, arg1);
1332     } else {
1333         TCGv_i64 t0 = tcg_const_i64(arg2);
1334         tcg_gen_or_i64(ret, arg1, t0);
1335         tcg_temp_free_i64(t0);
1336     }
1339 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1341     if (TCG_TARGET_REG_BITS == 32) {
1342         tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1343         tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1344         return;
1345     }
1346     /* Some cases can be optimized here.  */
1347     if (arg2 == 0) {
1348         tcg_gen_mov_i64(ret, arg1);
1349     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1350         /* Don't recurse with tcg_gen_not_i64.  */
1351         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1352     } else {
1353         TCGv_i64 t0 = tcg_const_i64(arg2);
1354         tcg_gen_xor_i64(ret, arg1, t0);
1355         tcg_temp_free_i64(t0);
1356     }
1359 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1360                                       unsigned c, bool right, bool arith)
1362     tcg_debug_assert(c < 64);
1363     if (c == 0) {
1364         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1365         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1366     } else if (c >= 32) {
1367         c -= 32;
1368         if (right) {
1369             if (arith) {
1370                 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1371                 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1372             } else {
1373                 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1374                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1375             }
1376         } else {
1377             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1378             tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1379         }
1380     } else if (right) {
1381         if (TCG_TARGET_HAS_extract2_i32) {
1382             tcg_gen_extract2_i32(TCGV_LOW(ret),
1383                                  TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1384         } else {
1385             tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1386             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1387                                 TCGV_HIGH(arg1), 32 - c, c);
1388         }
1389         if (arith) {
1390             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1391         } else {
1392             tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1393         }
1394     } else {
1395         if (TCG_TARGET_HAS_extract2_i32) {
1396             tcg_gen_extract2_i32(TCGV_HIGH(ret),
1397                                  TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
1398         } else {
1399             TCGv_i32 t0 = tcg_temp_new_i32();
1400             tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1401             tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1402                                 TCGV_HIGH(arg1), c, 32 - c);
1403             tcg_temp_free_i32(t0);
1404         }
1405         tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1406     }
1409 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1411     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1412     if (TCG_TARGET_REG_BITS == 32) {
1413         tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1414     } else if (arg2 == 0) {
1415         tcg_gen_mov_i64(ret, arg1);
1416     } else {
1417         TCGv_i64 t0 = tcg_const_i64(arg2);
1418         tcg_gen_shl_i64(ret, arg1, t0);
1419         tcg_temp_free_i64(t0);
1420     }
1423 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1425     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1426     if (TCG_TARGET_REG_BITS == 32) {
1427         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1428     } else if (arg2 == 0) {
1429         tcg_gen_mov_i64(ret, arg1);
1430     } else {
1431         TCGv_i64 t0 = tcg_const_i64(arg2);
1432         tcg_gen_shr_i64(ret, arg1, t0);
1433         tcg_temp_free_i64(t0);
1434     }
1437 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1439     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1440     if (TCG_TARGET_REG_BITS == 32) {
1441         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1442     } else if (arg2 == 0) {
1443         tcg_gen_mov_i64(ret, arg1);
1444     } else {
1445         TCGv_i64 t0 = tcg_const_i64(arg2);
1446         tcg_gen_sar_i64(ret, arg1, t0);
1447         tcg_temp_free_i64(t0);
1448     }
1451 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1453     if (cond == TCG_COND_ALWAYS) {
1454         tcg_gen_br(l);
1455     } else if (cond != TCG_COND_NEVER) {
1456         l->refs++;
1457         if (TCG_TARGET_REG_BITS == 32) {
1458             tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1459                               TCGV_HIGH(arg1), TCGV_LOW(arg2),
1460                               TCGV_HIGH(arg2), cond, label_arg(l));
1461         } else {
1462             tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1463                               label_arg(l));
1464         }
1465     }
1468 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1470     if (cond == TCG_COND_ALWAYS) {
1471         tcg_gen_br(l);
1472     } else if (cond != TCG_COND_NEVER) {
1473         TCGv_i64 t0 = tcg_const_i64(arg2);
1474         tcg_gen_brcond_i64(cond, arg1, t0, l);
1475         tcg_temp_free_i64(t0);
1476     }
1479 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1480                          TCGv_i64 arg1, TCGv_i64 arg2)
1482     if (cond == TCG_COND_ALWAYS) {
1483         tcg_gen_movi_i64(ret, 1);
1484     } else if (cond == TCG_COND_NEVER) {
1485         tcg_gen_movi_i64(ret, 0);
1486     } else {
1487         if (TCG_TARGET_REG_BITS == 32) {
1488             tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1489                              TCGV_LOW(arg1), TCGV_HIGH(arg1),
1490                              TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1491             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1492         } else {
1493             tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1494         }
1495     }
1498 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1499                           TCGv_i64 arg1, int64_t arg2)
1501     TCGv_i64 t0 = tcg_const_i64(arg2);
1502     tcg_gen_setcond_i64(cond, ret, arg1, t0);
1503     tcg_temp_free_i64(t0);
1506 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1508     if (arg2 == 0) {
1509         tcg_gen_movi_i64(ret, 0);
1510     } else if (is_power_of_2(arg2)) {
1511         tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1512     } else {
1513         TCGv_i64 t0 = tcg_const_i64(arg2);
1514         tcg_gen_mul_i64(ret, arg1, t0);
1515         tcg_temp_free_i64(t0);
1516     }
1519 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1521     if (TCG_TARGET_HAS_div_i64) {
1522         tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1523     } else if (TCG_TARGET_HAS_div2_i64) {
1524         TCGv_i64 t0 = tcg_temp_new_i64();
1525         tcg_gen_sari_i64(t0, arg1, 63);
1526         tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1527         tcg_temp_free_i64(t0);
1528     } else {
1529         gen_helper_div_i64(ret, arg1, arg2);
1530     }
1533 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1535     if (TCG_TARGET_HAS_rem_i64) {
1536         tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1537     } else if (TCG_TARGET_HAS_div_i64) {
1538         TCGv_i64 t0 = tcg_temp_new_i64();
1539         tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1540         tcg_gen_mul_i64(t0, t0, arg2);
1541         tcg_gen_sub_i64(ret, arg1, t0);
1542         tcg_temp_free_i64(t0);
1543     } else if (TCG_TARGET_HAS_div2_i64) {
1544         TCGv_i64 t0 = tcg_temp_new_i64();
1545         tcg_gen_sari_i64(t0, arg1, 63);
1546         tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1547         tcg_temp_free_i64(t0);
1548     } else {
1549         gen_helper_rem_i64(ret, arg1, arg2);
1550     }
1553 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1555     if (TCG_TARGET_HAS_div_i64) {
1556         tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1557     } else if (TCG_TARGET_HAS_div2_i64) {
1558         TCGv_i64 t0 = tcg_temp_new_i64();
1559         tcg_gen_movi_i64(t0, 0);
1560         tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1561         tcg_temp_free_i64(t0);
1562     } else {
1563         gen_helper_divu_i64(ret, arg1, arg2);
1564     }
1567 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1569     if (TCG_TARGET_HAS_rem_i64) {
1570         tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1571     } else if (TCG_TARGET_HAS_div_i64) {
1572         TCGv_i64 t0 = tcg_temp_new_i64();
1573         tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1574         tcg_gen_mul_i64(t0, t0, arg2);
1575         tcg_gen_sub_i64(ret, arg1, t0);
1576         tcg_temp_free_i64(t0);
1577     } else if (TCG_TARGET_HAS_div2_i64) {
1578         TCGv_i64 t0 = tcg_temp_new_i64();
1579         tcg_gen_movi_i64(t0, 0);
1580         tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1581         tcg_temp_free_i64(t0);
1582     } else {
1583         gen_helper_remu_i64(ret, arg1, arg2);
1584     }
1587 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1589     if (TCG_TARGET_REG_BITS == 32) {
1590         tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1591         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1592     } else if (TCG_TARGET_HAS_ext8s_i64) {
1593         tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1594     } else {
1595         tcg_gen_shli_i64(ret, arg, 56);
1596         tcg_gen_sari_i64(ret, ret, 56);
1597     }
1600 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1602     if (TCG_TARGET_REG_BITS == 32) {
1603         tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1604         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1605     } else if (TCG_TARGET_HAS_ext16s_i64) {
1606         tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1607     } else {
1608         tcg_gen_shli_i64(ret, arg, 48);
1609         tcg_gen_sari_i64(ret, ret, 48);
1610     }
1613 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1615     if (TCG_TARGET_REG_BITS == 32) {
1616         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1617         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1618     } else if (TCG_TARGET_HAS_ext32s_i64) {
1619         tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1620     } else {
1621         tcg_gen_shli_i64(ret, arg, 32);
1622         tcg_gen_sari_i64(ret, ret, 32);
1623     }
1626 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1628     if (TCG_TARGET_REG_BITS == 32) {
1629         tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1630         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1631     } else if (TCG_TARGET_HAS_ext8u_i64) {
1632         tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1633     } else {
1634         tcg_gen_andi_i64(ret, arg, 0xffu);
1635     }
1638 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1640     if (TCG_TARGET_REG_BITS == 32) {
1641         tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1642         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1643     } else if (TCG_TARGET_HAS_ext16u_i64) {
1644         tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1645     } else {
1646         tcg_gen_andi_i64(ret, arg, 0xffffu);
1647     }
1650 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1652     if (TCG_TARGET_REG_BITS == 32) {
1653         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1654         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1655     } else if (TCG_TARGET_HAS_ext32u_i64) {
1656         tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1657     } else {
1658         tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1659     }
1662 /* Note: we assume the six high bytes are set to zero */
1663 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1665     if (TCG_TARGET_REG_BITS == 32) {
1666         tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1667         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1668     } else if (TCG_TARGET_HAS_bswap16_i64) {
1669         tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1670     } else {
1671         TCGv_i64 t0 = tcg_temp_new_i64();
1673         tcg_gen_ext8u_i64(t0, arg);
1674         tcg_gen_shli_i64(t0, t0, 8);
1675         tcg_gen_shri_i64(ret, arg, 8);
1676         tcg_gen_or_i64(ret, ret, t0);
1677         tcg_temp_free_i64(t0);
1678     }
1681 /* Note: we assume the four high bytes are set to zero */
1682 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1684     if (TCG_TARGET_REG_BITS == 32) {
1685         tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1686         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1687     } else if (TCG_TARGET_HAS_bswap32_i64) {
1688         tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1689     } else {
1690         TCGv_i64 t0 = tcg_temp_new_i64();
1691         TCGv_i64 t1 = tcg_temp_new_i64();
1692         TCGv_i64 t2 = tcg_const_i64(0x00ff00ff);
1694                                         /* arg = ....abcd */
1695         tcg_gen_shri_i64(t0, arg, 8);   /*  t0 = .....abc */
1696         tcg_gen_and_i64(t1, arg, t2);   /*  t1 = .....b.d */
1697         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = .....a.c */
1698         tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = ....b.d. */
1699         tcg_gen_or_i64(ret, t0, t1);    /* ret = ....badc */
1701         tcg_gen_shli_i64(t1, ret, 48);  /*  t1 = dc...... */
1702         tcg_gen_shri_i64(t0, ret, 16);  /*  t0 = ......ba */
1703         tcg_gen_shri_i64(t1, t1, 32);   /*  t1 = ....dc.. */
1704         tcg_gen_or_i64(ret, t0, t1);    /* ret = ....dcba */
1706         tcg_temp_free_i64(t0);
1707         tcg_temp_free_i64(t1);
1708         tcg_temp_free_i64(t2);
1709     }
1712 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1714     if (TCG_TARGET_REG_BITS == 32) {
1715         TCGv_i32 t0, t1;
1716         t0 = tcg_temp_new_i32();
1717         t1 = tcg_temp_new_i32();
1719         tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1720         tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1721         tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1722         tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1723         tcg_temp_free_i32(t0);
1724         tcg_temp_free_i32(t1);
1725     } else if (TCG_TARGET_HAS_bswap64_i64) {
1726         tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1727     } else {
1728         TCGv_i64 t0 = tcg_temp_new_i64();
1729         TCGv_i64 t1 = tcg_temp_new_i64();
1730         TCGv_i64 t2 = tcg_temp_new_i64();
1732                                         /* arg = abcdefgh */
1733         tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1734         tcg_gen_shri_i64(t0, arg, 8);   /*  t0 = .abcdefg */
1735         tcg_gen_and_i64(t1, arg, t2);   /*  t1 = .b.d.f.h */
1736         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = .a.c.e.g */
1737         tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = b.d.f.h. */
1738         tcg_gen_or_i64(ret, t0, t1);    /* ret = badcfehg */
1740         tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1741         tcg_gen_shri_i64(t0, ret, 16);  /*  t0 = ..badcfe */
1742         tcg_gen_and_i64(t1, ret, t2);   /*  t1 = ..dc..hg */
1743         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = ..ba..fe */
1744         tcg_gen_shli_i64(t1, t1, 16);   /*  t1 = dc..hg.. */
1745         tcg_gen_or_i64(ret, t0, t1);    /* ret = dcbahgfe */
1747         tcg_gen_shri_i64(t0, ret, 32);  /*  t0 = ....dcba */
1748         tcg_gen_shli_i64(t1, ret, 32);  /*  t1 = hgfe.... */
1749         tcg_gen_or_i64(ret, t0, t1);    /* ret = hgfedcba */
1751         tcg_temp_free_i64(t0);
1752         tcg_temp_free_i64(t1);
1753         tcg_temp_free_i64(t2);
1754     }
1757 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1759     if (TCG_TARGET_REG_BITS == 32) {
1760         tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1761         tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1762     } else if (TCG_TARGET_HAS_not_i64) {
1763         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1764     } else {
1765         tcg_gen_xori_i64(ret, arg, -1);
1766     }
1769 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1771     if (TCG_TARGET_REG_BITS == 32) {
1772         tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1773         tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1774     } else if (TCG_TARGET_HAS_andc_i64) {
1775         tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1776     } else {
1777         TCGv_i64 t0 = tcg_temp_new_i64();
1778         tcg_gen_not_i64(t0, arg2);
1779         tcg_gen_and_i64(ret, arg1, t0);
1780         tcg_temp_free_i64(t0);
1781     }
1784 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1786     if (TCG_TARGET_REG_BITS == 32) {
1787         tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1788         tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1789     } else if (TCG_TARGET_HAS_eqv_i64) {
1790         tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1791     } else {
1792         tcg_gen_xor_i64(ret, arg1, arg2);
1793         tcg_gen_not_i64(ret, ret);
1794     }
1797 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1799     if (TCG_TARGET_REG_BITS == 32) {
1800         tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1801         tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1802     } else if (TCG_TARGET_HAS_nand_i64) {
1803         tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1804     } else {
1805         tcg_gen_and_i64(ret, arg1, arg2);
1806         tcg_gen_not_i64(ret, ret);
1807     }
1810 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1812     if (TCG_TARGET_REG_BITS == 32) {
1813         tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1814         tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1815     } else if (TCG_TARGET_HAS_nor_i64) {
1816         tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1817     } else {
1818         tcg_gen_or_i64(ret, arg1, arg2);
1819         tcg_gen_not_i64(ret, ret);
1820     }
1823 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1825     if (TCG_TARGET_REG_BITS == 32) {
1826         tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1827         tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1828     } else if (TCG_TARGET_HAS_orc_i64) {
1829         tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1830     } else {
1831         TCGv_i64 t0 = tcg_temp_new_i64();
1832         tcg_gen_not_i64(t0, arg2);
1833         tcg_gen_or_i64(ret, arg1, t0);
1834         tcg_temp_free_i64(t0);
1835     }
1838 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1840     if (TCG_TARGET_HAS_clz_i64) {
1841         tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1842     } else {
1843         gen_helper_clz_i64(ret, arg1, arg2);
1844     }
1847 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1849     if (TCG_TARGET_REG_BITS == 32
1850         && TCG_TARGET_HAS_clz_i32
1851         && arg2 <= 0xffffffffu) {
1852         TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1853         tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1854         tcg_gen_addi_i32(t, t, 32);
1855         tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1856         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1857         tcg_temp_free_i32(t);
1858     } else {
1859         TCGv_i64 t = tcg_const_i64(arg2);
1860         tcg_gen_clz_i64(ret, arg1, t);
1861         tcg_temp_free_i64(t);
1862     }
1865 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1867     if (TCG_TARGET_HAS_ctz_i64) {
1868         tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1869     } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1870         TCGv_i64 z, t = tcg_temp_new_i64();
1872         if (TCG_TARGET_HAS_ctpop_i64) {
1873             tcg_gen_subi_i64(t, arg1, 1);
1874             tcg_gen_andc_i64(t, t, arg1);
1875             tcg_gen_ctpop_i64(t, t);
1876         } else {
1877             /* Since all non-x86 hosts have clz(0) == 64, don't fight it.  */
1878             tcg_gen_neg_i64(t, arg1);
1879             tcg_gen_and_i64(t, t, arg1);
1880             tcg_gen_clzi_i64(t, t, 64);
1881             tcg_gen_xori_i64(t, t, 63);
1882         }
1883         z = tcg_const_i64(0);
1884         tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1885         tcg_temp_free_i64(t);
1886         tcg_temp_free_i64(z);
1887     } else {
1888         gen_helper_ctz_i64(ret, arg1, arg2);
1889     }
1892 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1894     if (TCG_TARGET_REG_BITS == 32
1895         && TCG_TARGET_HAS_ctz_i32
1896         && arg2 <= 0xffffffffu) {
1897         TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1898         tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1899         tcg_gen_addi_i32(t32, t32, 32);
1900         tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1901         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1902         tcg_temp_free_i32(t32);
1903     } else if (!TCG_TARGET_HAS_ctz_i64
1904                && TCG_TARGET_HAS_ctpop_i64
1905                && arg2 == 64) {
1906         /* This equivalence has the advantage of not requiring a fixup.  */
1907         TCGv_i64 t = tcg_temp_new_i64();
1908         tcg_gen_subi_i64(t, arg1, 1);
1909         tcg_gen_andc_i64(t, t, arg1);
1910         tcg_gen_ctpop_i64(ret, t);
1911         tcg_temp_free_i64(t);
1912     } else {
1913         TCGv_i64 t64 = tcg_const_i64(arg2);
1914         tcg_gen_ctz_i64(ret, arg1, t64);
1915         tcg_temp_free_i64(t64);
1916     }
1919 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1921     if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1922         TCGv_i64 t = tcg_temp_new_i64();
1923         tcg_gen_sari_i64(t, arg, 63);
1924         tcg_gen_xor_i64(t, t, arg);
1925         tcg_gen_clzi_i64(t, t, 64);
1926         tcg_gen_subi_i64(ret, t, 1);
1927         tcg_temp_free_i64(t);
1928     } else {
1929         gen_helper_clrsb_i64(ret, arg);
1930     }
1933 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1935     if (TCG_TARGET_HAS_ctpop_i64) {
1936         tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1937     } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1938         tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1939         tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1940         tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1941         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1942     } else {
1943         gen_helper_ctpop_i64(ret, arg1);
1944     }
1947 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1949     if (TCG_TARGET_HAS_rot_i64) {
1950         tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1951     } else {
1952         TCGv_i64 t0, t1;
1953         t0 = tcg_temp_new_i64();
1954         t1 = tcg_temp_new_i64();
1955         tcg_gen_shl_i64(t0, arg1, arg2);
1956         tcg_gen_subfi_i64(t1, 64, arg2);
1957         tcg_gen_shr_i64(t1, arg1, t1);
1958         tcg_gen_or_i64(ret, t0, t1);
1959         tcg_temp_free_i64(t0);
1960         tcg_temp_free_i64(t1);
1961     }
1964 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1966     tcg_debug_assert(arg2 < 64);
1967     /* some cases can be optimized here */
1968     if (arg2 == 0) {
1969         tcg_gen_mov_i64(ret, arg1);
1970     } else if (TCG_TARGET_HAS_rot_i64) {
1971         TCGv_i64 t0 = tcg_const_i64(arg2);
1972         tcg_gen_rotl_i64(ret, arg1, t0);
1973         tcg_temp_free_i64(t0);
1974     } else {
1975         TCGv_i64 t0, t1;
1976         t0 = tcg_temp_new_i64();
1977         t1 = tcg_temp_new_i64();
1978         tcg_gen_shli_i64(t0, arg1, arg2);
1979         tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1980         tcg_gen_or_i64(ret, t0, t1);
1981         tcg_temp_free_i64(t0);
1982         tcg_temp_free_i64(t1);
1983     }
1986 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1988     if (TCG_TARGET_HAS_rot_i64) {
1989         tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1990     } else {
1991         TCGv_i64 t0, t1;
1992         t0 = tcg_temp_new_i64();
1993         t1 = tcg_temp_new_i64();
1994         tcg_gen_shr_i64(t0, arg1, arg2);
1995         tcg_gen_subfi_i64(t1, 64, arg2);
1996         tcg_gen_shl_i64(t1, arg1, t1);
1997         tcg_gen_or_i64(ret, t0, t1);
1998         tcg_temp_free_i64(t0);
1999         tcg_temp_free_i64(t1);
2000     }
2003 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
2005     tcg_debug_assert(arg2 < 64);
2006     /* some cases can be optimized here */
2007     if (arg2 == 0) {
2008         tcg_gen_mov_i64(ret, arg1);
2009     } else {
2010         tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2011     }
2014 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2015                          unsigned int ofs, unsigned int len)
2017     uint64_t mask;
2018     TCGv_i64 t1;
2020     tcg_debug_assert(ofs < 64);
2021     tcg_debug_assert(len > 0);
2022     tcg_debug_assert(len <= 64);
2023     tcg_debug_assert(ofs + len <= 64);
2025     if (len == 64) {
2026         tcg_gen_mov_i64(ret, arg2);
2027         return;
2028     }
2029     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2030         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2031         return;
2032     }
2034     if (TCG_TARGET_REG_BITS == 32) {
2035         if (ofs >= 32) {
2036             tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2037                                 TCGV_LOW(arg2), ofs - 32, len);
2038             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2039             return;
2040         }
2041         if (ofs + len <= 32) {
2042             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2043                                 TCGV_LOW(arg2), ofs, len);
2044             tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2045             return;
2046         }
2047     }
2049     t1 = tcg_temp_new_i64();
2051     if (TCG_TARGET_HAS_extract2_i64) {
2052         if (ofs + len == 64) {
2053             tcg_gen_shli_i64(t1, arg1, len);
2054             tcg_gen_extract2_i64(ret, t1, arg2, len);
2055             goto done;
2056         }
2057         if (ofs == 0) {
2058             tcg_gen_extract2_i64(ret, arg1, arg2, len);
2059             tcg_gen_rotli_i64(ret, ret, len);
2060             goto done;
2061         }
2062     }
2064     mask = (1ull << len) - 1;
2065     if (ofs + len < 64) {
2066         tcg_gen_andi_i64(t1, arg2, mask);
2067         tcg_gen_shli_i64(t1, t1, ofs);
2068     } else {
2069         tcg_gen_shli_i64(t1, arg2, ofs);
2070     }
2071     tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2072     tcg_gen_or_i64(ret, ret, t1);
2073  done:
2074     tcg_temp_free_i64(t1);
2077 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2078                            unsigned int ofs, unsigned int len)
2080     tcg_debug_assert(ofs < 64);
2081     tcg_debug_assert(len > 0);
2082     tcg_debug_assert(len <= 64);
2083     tcg_debug_assert(ofs + len <= 64);
2085     if (ofs + len == 64) {
2086         tcg_gen_shli_i64(ret, arg, ofs);
2087     } else if (ofs == 0) {
2088         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2089     } else if (TCG_TARGET_HAS_deposit_i64
2090                && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2091         TCGv_i64 zero = tcg_const_i64(0);
2092         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2093         tcg_temp_free_i64(zero);
2094     } else {
2095         if (TCG_TARGET_REG_BITS == 32) {
2096             if (ofs >= 32) {
2097                 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2098                                       ofs - 32, len);
2099                 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2100                 return;
2101             }
2102             if (ofs + len <= 32) {
2103                 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2104                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2105                 return;
2106             }
2107         }
2108         /* To help two-operand hosts we prefer to zero-extend first,
2109            which allows ARG to stay live.  */
2110         switch (len) {
2111         case 32:
2112             if (TCG_TARGET_HAS_ext32u_i64) {
2113                 tcg_gen_ext32u_i64(ret, arg);
2114                 tcg_gen_shli_i64(ret, ret, ofs);
2115                 return;
2116             }
2117             break;
2118         case 16:
2119             if (TCG_TARGET_HAS_ext16u_i64) {
2120                 tcg_gen_ext16u_i64(ret, arg);
2121                 tcg_gen_shli_i64(ret, ret, ofs);
2122                 return;
2123             }
2124             break;
2125         case 8:
2126             if (TCG_TARGET_HAS_ext8u_i64) {
2127                 tcg_gen_ext8u_i64(ret, arg);
2128                 tcg_gen_shli_i64(ret, ret, ofs);
2129                 return;
2130             }
2131             break;
2132         }
2133         /* Otherwise prefer zero-extension over AND for code size.  */
2134         switch (ofs + len) {
2135         case 32:
2136             if (TCG_TARGET_HAS_ext32u_i64) {
2137                 tcg_gen_shli_i64(ret, arg, ofs);
2138                 tcg_gen_ext32u_i64(ret, ret);
2139                 return;
2140             }
2141             break;
2142         case 16:
2143             if (TCG_TARGET_HAS_ext16u_i64) {
2144                 tcg_gen_shli_i64(ret, arg, ofs);
2145                 tcg_gen_ext16u_i64(ret, ret);
2146                 return;
2147             }
2148             break;
2149         case 8:
2150             if (TCG_TARGET_HAS_ext8u_i64) {
2151                 tcg_gen_shli_i64(ret, arg, ofs);
2152                 tcg_gen_ext8u_i64(ret, ret);
2153                 return;
2154             }
2155             break;
2156         }
2157         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2158         tcg_gen_shli_i64(ret, ret, ofs);
2159     }
2162 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2163                          unsigned int ofs, unsigned int len)
2165     tcg_debug_assert(ofs < 64);
2166     tcg_debug_assert(len > 0);
2167     tcg_debug_assert(len <= 64);
2168     tcg_debug_assert(ofs + len <= 64);
2170     /* Canonicalize certain special cases, even if extract is supported.  */
2171     if (ofs + len == 64) {
2172         tcg_gen_shri_i64(ret, arg, 64 - len);
2173         return;
2174     }
2175     if (ofs == 0) {
2176         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2177         return;
2178     }
2180     if (TCG_TARGET_REG_BITS == 32) {
2181         /* Look for a 32-bit extract within one of the two words.  */
2182         if (ofs >= 32) {
2183             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2184             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2185             return;
2186         }
2187         if (ofs + len <= 32) {
2188             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2189             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2190             return;
2191         }
2192         /* The field is split across two words.  One double-word
2193            shift is better than two double-word shifts.  */
2194         goto do_shift_and;
2195     }
2197     if (TCG_TARGET_HAS_extract_i64
2198         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2199         tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2200         return;
2201     }
2203     /* Assume that zero-extension, if available, is cheaper than a shift.  */
2204     switch (ofs + len) {
2205     case 32:
2206         if (TCG_TARGET_HAS_ext32u_i64) {
2207             tcg_gen_ext32u_i64(ret, arg);
2208             tcg_gen_shri_i64(ret, ret, ofs);
2209             return;
2210         }
2211         break;
2212     case 16:
2213         if (TCG_TARGET_HAS_ext16u_i64) {
2214             tcg_gen_ext16u_i64(ret, arg);
2215             tcg_gen_shri_i64(ret, ret, ofs);
2216             return;
2217         }
2218         break;
2219     case 8:
2220         if (TCG_TARGET_HAS_ext8u_i64) {
2221             tcg_gen_ext8u_i64(ret, arg);
2222             tcg_gen_shri_i64(ret, ret, ofs);
2223             return;
2224         }
2225         break;
2226     }
2228     /* ??? Ideally we'd know what values are available for immediate AND.
2229        Assume that 8 bits are available, plus the special cases of 16 and 32,
2230        so that we get ext8u, ext16u, and ext32u.  */
2231     switch (len) {
2232     case 1 ... 8: case 16: case 32:
2233     do_shift_and:
2234         tcg_gen_shri_i64(ret, arg, ofs);
2235         tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2236         break;
2237     default:
2238         tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2239         tcg_gen_shri_i64(ret, ret, 64 - len);
2240         break;
2241     }
2244 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2245                           unsigned int ofs, unsigned int len)
2247     tcg_debug_assert(ofs < 64);
2248     tcg_debug_assert(len > 0);
2249     tcg_debug_assert(len <= 64);
2250     tcg_debug_assert(ofs + len <= 64);
2252     /* Canonicalize certain special cases, even if sextract is supported.  */
2253     if (ofs + len == 64) {
2254         tcg_gen_sari_i64(ret, arg, 64 - len);
2255         return;
2256     }
2257     if (ofs == 0) {
2258         switch (len) {
2259         case 32:
2260             tcg_gen_ext32s_i64(ret, arg);
2261             return;
2262         case 16:
2263             tcg_gen_ext16s_i64(ret, arg);
2264             return;
2265         case 8:
2266             tcg_gen_ext8s_i64(ret, arg);
2267             return;
2268         }
2269     }
2271     if (TCG_TARGET_REG_BITS == 32) {
2272         /* Look for a 32-bit extract within one of the two words.  */
2273         if (ofs >= 32) {
2274             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2275         } else if (ofs + len <= 32) {
2276             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2277         } else if (ofs == 0) {
2278             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2279             tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2280             return;
2281         } else if (len > 32) {
2282             TCGv_i32 t = tcg_temp_new_i32();
2283             /* Extract the bits for the high word normally.  */
2284             tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2285             /* Shift the field down for the low part.  */
2286             tcg_gen_shri_i64(ret, arg, ofs);
2287             /* Overwrite the shift into the high part.  */
2288             tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2289             tcg_temp_free_i32(t);
2290             return;
2291         } else {
2292             /* Shift the field down for the low part, such that the
2293                field sits at the MSB.  */
2294             tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2295             /* Shift the field down from the MSB, sign extending.  */
2296             tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2297         }
2298         /* Sign-extend the field from 32 bits.  */
2299         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2300         return;
2301     }
2303     if (TCG_TARGET_HAS_sextract_i64
2304         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2305         tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2306         return;
2307     }
2309     /* Assume that sign-extension, if available, is cheaper than a shift.  */
2310     switch (ofs + len) {
2311     case 32:
2312         if (TCG_TARGET_HAS_ext32s_i64) {
2313             tcg_gen_ext32s_i64(ret, arg);
2314             tcg_gen_sari_i64(ret, ret, ofs);
2315             return;
2316         }
2317         break;
2318     case 16:
2319         if (TCG_TARGET_HAS_ext16s_i64) {
2320             tcg_gen_ext16s_i64(ret, arg);
2321             tcg_gen_sari_i64(ret, ret, ofs);
2322             return;
2323         }
2324         break;
2325     case 8:
2326         if (TCG_TARGET_HAS_ext8s_i64) {
2327             tcg_gen_ext8s_i64(ret, arg);
2328             tcg_gen_sari_i64(ret, ret, ofs);
2329             return;
2330         }
2331         break;
2332     }
2333     switch (len) {
2334     case 32:
2335         if (TCG_TARGET_HAS_ext32s_i64) {
2336             tcg_gen_shri_i64(ret, arg, ofs);
2337             tcg_gen_ext32s_i64(ret, ret);
2338             return;
2339         }
2340         break;
2341     case 16:
2342         if (TCG_TARGET_HAS_ext16s_i64) {
2343             tcg_gen_shri_i64(ret, arg, ofs);
2344             tcg_gen_ext16s_i64(ret, ret);
2345             return;
2346         }
2347         break;
2348     case 8:
2349         if (TCG_TARGET_HAS_ext8s_i64) {
2350             tcg_gen_shri_i64(ret, arg, ofs);
2351             tcg_gen_ext8s_i64(ret, ret);
2352             return;
2353         }
2354         break;
2355     }
2356     tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2357     tcg_gen_sari_i64(ret, ret, 64 - len);
2361  * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2362  * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2363  */
2364 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2365                           unsigned int ofs)
2367     tcg_debug_assert(ofs <= 64);
2368     if (ofs == 0) {
2369         tcg_gen_mov_i64(ret, al);
2370     } else if (ofs == 64) {
2371         tcg_gen_mov_i64(ret, ah);
2372     } else if (al == ah) {
2373         tcg_gen_rotri_i64(ret, al, ofs);
2374     } else if (TCG_TARGET_HAS_extract2_i64) {
2375         tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2376     } else {
2377         TCGv_i64 t0 = tcg_temp_new_i64();
2378         tcg_gen_shri_i64(t0, al, ofs);
2379         tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2380         tcg_temp_free_i64(t0);
2381     }
2384 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2385                          TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2387     if (cond == TCG_COND_ALWAYS) {
2388         tcg_gen_mov_i64(ret, v1);
2389     } else if (cond == TCG_COND_NEVER) {
2390         tcg_gen_mov_i64(ret, v2);
2391     } else if (TCG_TARGET_REG_BITS == 32) {
2392         TCGv_i32 t0 = tcg_temp_new_i32();
2393         TCGv_i32 t1 = tcg_temp_new_i32();
2394         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2395                          TCGV_LOW(c1), TCGV_HIGH(c1),
2396                          TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2398         if (TCG_TARGET_HAS_movcond_i32) {
2399             tcg_gen_movi_i32(t1, 0);
2400             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2401                                 TCGV_LOW(v1), TCGV_LOW(v2));
2402             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2403                                 TCGV_HIGH(v1), TCGV_HIGH(v2));
2404         } else {
2405             tcg_gen_neg_i32(t0, t0);
2407             tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2408             tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2409             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2411             tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2412             tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2413             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2414         }
2415         tcg_temp_free_i32(t0);
2416         tcg_temp_free_i32(t1);
2417     } else if (TCG_TARGET_HAS_movcond_i64) {
2418         tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2419     } else {
2420         TCGv_i64 t0 = tcg_temp_new_i64();
2421         TCGv_i64 t1 = tcg_temp_new_i64();
2422         tcg_gen_setcond_i64(cond, t0, c1, c2);
2423         tcg_gen_neg_i64(t0, t0);
2424         tcg_gen_and_i64(t1, v1, t0);
2425         tcg_gen_andc_i64(ret, v2, t0);
2426         tcg_gen_or_i64(ret, ret, t1);
2427         tcg_temp_free_i64(t0);
2428         tcg_temp_free_i64(t1);
2429     }
2432 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2433                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2435     if (TCG_TARGET_HAS_add2_i64) {
2436         tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2437     } else {
2438         TCGv_i64 t0 = tcg_temp_new_i64();
2439         TCGv_i64 t1 = tcg_temp_new_i64();
2440         tcg_gen_add_i64(t0, al, bl);
2441         tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2442         tcg_gen_add_i64(rh, ah, bh);
2443         tcg_gen_add_i64(rh, rh, t1);
2444         tcg_gen_mov_i64(rl, t0);
2445         tcg_temp_free_i64(t0);
2446         tcg_temp_free_i64(t1);
2447     }
2450 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2451                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2453     if (TCG_TARGET_HAS_sub2_i64) {
2454         tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2455     } else {
2456         TCGv_i64 t0 = tcg_temp_new_i64();
2457         TCGv_i64 t1 = tcg_temp_new_i64();
2458         tcg_gen_sub_i64(t0, al, bl);
2459         tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2460         tcg_gen_sub_i64(rh, ah, bh);
2461         tcg_gen_sub_i64(rh, rh, t1);
2462         tcg_gen_mov_i64(rl, t0);
2463         tcg_temp_free_i64(t0);
2464         tcg_temp_free_i64(t1);
2465     }
2468 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2470     if (TCG_TARGET_HAS_mulu2_i64) {
2471         tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2472     } else if (TCG_TARGET_HAS_muluh_i64) {
2473         TCGv_i64 t = tcg_temp_new_i64();
2474         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2475         tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2476         tcg_gen_mov_i64(rl, t);
2477         tcg_temp_free_i64(t);
2478     } else {
2479         TCGv_i64 t0 = tcg_temp_new_i64();
2480         tcg_gen_mul_i64(t0, arg1, arg2);
2481         gen_helper_muluh_i64(rh, arg1, arg2);
2482         tcg_gen_mov_i64(rl, t0);
2483         tcg_temp_free_i64(t0);
2484     }
2487 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2489     if (TCG_TARGET_HAS_muls2_i64) {
2490         tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2491     } else if (TCG_TARGET_HAS_mulsh_i64) {
2492         TCGv_i64 t = tcg_temp_new_i64();
2493         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2494         tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2495         tcg_gen_mov_i64(rl, t);
2496         tcg_temp_free_i64(t);
2497     } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2498         TCGv_i64 t0 = tcg_temp_new_i64();
2499         TCGv_i64 t1 = tcg_temp_new_i64();
2500         TCGv_i64 t2 = tcg_temp_new_i64();
2501         TCGv_i64 t3 = tcg_temp_new_i64();
2502         tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2503         /* Adjust for negative inputs.  */
2504         tcg_gen_sari_i64(t2, arg1, 63);
2505         tcg_gen_sari_i64(t3, arg2, 63);
2506         tcg_gen_and_i64(t2, t2, arg2);
2507         tcg_gen_and_i64(t3, t3, arg1);
2508         tcg_gen_sub_i64(rh, t1, t2);
2509         tcg_gen_sub_i64(rh, rh, t3);
2510         tcg_gen_mov_i64(rl, t0);
2511         tcg_temp_free_i64(t0);
2512         tcg_temp_free_i64(t1);
2513         tcg_temp_free_i64(t2);
2514         tcg_temp_free_i64(t3);
2515     } else {
2516         TCGv_i64 t0 = tcg_temp_new_i64();
2517         tcg_gen_mul_i64(t0, arg1, arg2);
2518         gen_helper_mulsh_i64(rh, arg1, arg2);
2519         tcg_gen_mov_i64(rl, t0);
2520         tcg_temp_free_i64(t0);
2521     }
2524 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2526     TCGv_i64 t0 = tcg_temp_new_i64();
2527     TCGv_i64 t1 = tcg_temp_new_i64();
2528     TCGv_i64 t2 = tcg_temp_new_i64();
2529     tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2530     /* Adjust for negative input for the signed arg1.  */
2531     tcg_gen_sari_i64(t2, arg1, 63);
2532     tcg_gen_and_i64(t2, t2, arg2);
2533     tcg_gen_sub_i64(rh, t1, t2);
2534     tcg_gen_mov_i64(rl, t0);
2535     tcg_temp_free_i64(t0);
2536     tcg_temp_free_i64(t1);
2537     tcg_temp_free_i64(t2);
2540 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2542     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2545 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2547     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2550 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2552     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2555 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2557     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2560 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2562     TCGv_i64 t = tcg_temp_new_i64();
2564     tcg_gen_sari_i64(t, a, 63);
2565     tcg_gen_xor_i64(ret, a, t);
2566     tcg_gen_sub_i64(ret, ret, t);
2567     tcg_temp_free_i64(t);
2570 /* Size changing operations.  */
2572 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2574     if (TCG_TARGET_REG_BITS == 32) {
2575         tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2576     } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2577         tcg_gen_op2(INDEX_op_extrl_i64_i32,
2578                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2579     } else {
2580         tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2581     }
2584 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2586     if (TCG_TARGET_REG_BITS == 32) {
2587         tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2588     } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2589         tcg_gen_op2(INDEX_op_extrh_i64_i32,
2590                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2591     } else {
2592         TCGv_i64 t = tcg_temp_new_i64();
2593         tcg_gen_shri_i64(t, arg, 32);
2594         tcg_gen_mov_i32(ret, (TCGv_i32)t);
2595         tcg_temp_free_i64(t);
2596     }
2599 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2601     if (TCG_TARGET_REG_BITS == 32) {
2602         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2603         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2604     } else {
2605         tcg_gen_op2(INDEX_op_extu_i32_i64,
2606                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2607     }
2610 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2612     if (TCG_TARGET_REG_BITS == 32) {
2613         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2614         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2615     } else {
2616         tcg_gen_op2(INDEX_op_ext_i32_i64,
2617                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2618     }
2621 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2623     TCGv_i64 tmp;
2625     if (TCG_TARGET_REG_BITS == 32) {
2626         tcg_gen_mov_i32(TCGV_LOW(dest), low);
2627         tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2628         return;
2629     }
2631     tmp = tcg_temp_new_i64();
2632     /* These extensions are only needed for type correctness.
2633        We may be able to do better given target specific information.  */
2634     tcg_gen_extu_i32_i64(tmp, high);
2635     tcg_gen_extu_i32_i64(dest, low);
2636     /* If deposit is available, use it.  Otherwise use the extra
2637        knowledge that we have of the zero-extensions above.  */
2638     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2639         tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2640     } else {
2641         tcg_gen_shli_i64(tmp, tmp, 32);
2642         tcg_gen_or_i64(dest, dest, tmp);
2643     }
2644     tcg_temp_free_i64(tmp);
2647 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2649     if (TCG_TARGET_REG_BITS == 32) {
2650         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2651         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2652     } else {
2653         tcg_gen_extrl_i64_i32(lo, arg);
2654         tcg_gen_extrh_i64_i32(hi, arg);
2655     }
2658 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2660     tcg_gen_ext32u_i64(lo, arg);
2661     tcg_gen_shri_i64(hi, arg, 32);
2664 /* QEMU specific operations.  */
2666 void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
2668     uintptr_t val = (uintptr_t)tb + idx;
2670     if (tb == NULL) {
2671         tcg_debug_assert(idx == 0);
2672     } else if (idx <= TB_EXIT_IDXMAX) {
2673 #ifdef CONFIG_DEBUG_TCG
2674         /* This is an exit following a goto_tb.  Verify that we have
2675            seen this numbered exit before, via tcg_gen_goto_tb.  */
2676         tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2677 #endif
2678         /* When not chaining, exit without indicating a link.  */
2679         if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2680             val = 0;
2681         }
2682     } else {
2683         /* This is an exit via the exitreq label.  */
2684         tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2685     }
2687     tcg_gen_op1i(INDEX_op_exit_tb, val);
2690 void tcg_gen_goto_tb(unsigned idx)
2692     /* We only support two chained exits.  */
2693     tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2694 #ifdef CONFIG_DEBUG_TCG
2695     /* Verify that we havn't seen this numbered exit before.  */
2696     tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2697     tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2698 #endif
2699     /* When not chaining, we simply fall through to the "fallback" exit.  */
2700     if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2701         tcg_gen_op1i(INDEX_op_goto_tb, idx);
2702     }
2705 void tcg_gen_lookup_and_goto_ptr(void)
2707     if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2708         TCGv_ptr ptr = tcg_temp_new_ptr();
2709         gen_helper_lookup_tb_ptr(ptr, cpu_env);
2710         tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2711         tcg_temp_free_ptr(ptr);
2712     } else {
2713         tcg_gen_exit_tb(NULL, 0);
2714     }
2717 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2719     /* Trigger the asserts within as early as possible.  */
2720     (void)get_alignment_bits(op);
2722     switch (op & MO_SIZE) {
2723     case MO_8:
2724         op &= ~MO_BSWAP;
2725         break;
2726     case MO_16:
2727         break;
2728     case MO_32:
2729         if (!is64) {
2730             op &= ~MO_SIGN;
2731         }
2732         break;
2733     case MO_64:
2734         if (!is64) {
2735             tcg_abort();
2736         }
2737         break;
2738     }
2739     if (st) {
2740         op &= ~MO_SIGN;
2741     }
2742     return op;
2745 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2746                          TCGMemOp memop, TCGArg idx)
2748     TCGMemOpIdx oi = make_memop_idx(memop, idx);
2749 #if TARGET_LONG_BITS == 32
2750     tcg_gen_op3i_i32(opc, val, addr, oi);
2751 #else
2752     if (TCG_TARGET_REG_BITS == 32) {
2753         tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2754     } else {
2755         tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2756     }
2757 #endif
2760 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2761                          TCGMemOp memop, TCGArg idx)
2763     TCGMemOpIdx oi = make_memop_idx(memop, idx);
2764 #if TARGET_LONG_BITS == 32
2765     if (TCG_TARGET_REG_BITS == 32) {
2766         tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2767     } else {
2768         tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2769     }
2770 #else
2771     if (TCG_TARGET_REG_BITS == 32) {
2772         tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2773                          TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2774     } else {
2775         tcg_gen_op3i_i64(opc, val, addr, oi);
2776     }
2777 #endif
2780 static void tcg_gen_req_mo(TCGBar type)
2782 #ifdef TCG_GUEST_DEFAULT_MO
2783     type &= TCG_GUEST_DEFAULT_MO;
2784 #endif
2785     type &= ~TCG_TARGET_DEFAULT_MO;
2786     if (type) {
2787         tcg_gen_mb(type | TCG_BAR_SC);
2788     }
2791 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2793     TCGMemOp orig_memop;
2795     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2796     memop = tcg_canonicalize_memop(memop, 0, 0);
2797     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2798                                addr, trace_mem_get_info(memop, 0));
2800     orig_memop = memop;
2801     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2802         memop &= ~MO_BSWAP;
2803         /* The bswap primitive requires zero-extended input.  */
2804         if ((memop & MO_SSIZE) == MO_SW) {
2805             memop &= ~MO_SIGN;
2806         }
2807     }
2809     gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2811     if ((orig_memop ^ memop) & MO_BSWAP) {
2812         switch (orig_memop & MO_SIZE) {
2813         case MO_16:
2814             tcg_gen_bswap16_i32(val, val);
2815             if (orig_memop & MO_SIGN) {
2816                 tcg_gen_ext16s_i32(val, val);
2817             }
2818             break;
2819         case MO_32:
2820             tcg_gen_bswap32_i32(val, val);
2821             break;
2822         default:
2823             g_assert_not_reached();
2824         }
2825     }
2828 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2830     TCGv_i32 swap = NULL;
2832     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2833     memop = tcg_canonicalize_memop(memop, 0, 1);
2834     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2835                                addr, trace_mem_get_info(memop, 1));
2837     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2838         swap = tcg_temp_new_i32();
2839         switch (memop & MO_SIZE) {
2840         case MO_16:
2841             tcg_gen_ext16u_i32(swap, val);
2842             tcg_gen_bswap16_i32(swap, swap);
2843             break;
2844         case MO_32:
2845             tcg_gen_bswap32_i32(swap, val);
2846             break;
2847         default:
2848             g_assert_not_reached();
2849         }
2850         val = swap;
2851         memop &= ~MO_BSWAP;
2852     }
2854     gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2856     if (swap) {
2857         tcg_temp_free_i32(swap);
2858     }
2861 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2863     TCGMemOp orig_memop;
2865     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2866         tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2867         if (memop & MO_SIGN) {
2868             tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2869         } else {
2870             tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2871         }
2872         return;
2873     }
2875     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2876     memop = tcg_canonicalize_memop(memop, 1, 0);
2877     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2878                                addr, trace_mem_get_info(memop, 0));
2880     orig_memop = memop;
2881     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2882         memop &= ~MO_BSWAP;
2883         /* The bswap primitive requires zero-extended input.  */
2884         if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
2885             memop &= ~MO_SIGN;
2886         }
2887     }
2889     gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2891     if ((orig_memop ^ memop) & MO_BSWAP) {
2892         switch (orig_memop & MO_SIZE) {
2893         case MO_16:
2894             tcg_gen_bswap16_i64(val, val);
2895             if (orig_memop & MO_SIGN) {
2896                 tcg_gen_ext16s_i64(val, val);
2897             }
2898             break;
2899         case MO_32:
2900             tcg_gen_bswap32_i64(val, val);
2901             if (orig_memop & MO_SIGN) {
2902                 tcg_gen_ext32s_i64(val, val);
2903             }
2904             break;
2905         case MO_64:
2906             tcg_gen_bswap64_i64(val, val);
2907             break;
2908         default:
2909             g_assert_not_reached();
2910         }
2911     }
2914 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2916     TCGv_i64 swap = NULL;
2918     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2919         tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2920         return;
2921     }
2923     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2924     memop = tcg_canonicalize_memop(memop, 1, 1);
2925     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2926                                addr, trace_mem_get_info(memop, 1));
2928     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2929         swap = tcg_temp_new_i64();
2930         switch (memop & MO_SIZE) {
2931         case MO_16:
2932             tcg_gen_ext16u_i64(swap, val);
2933             tcg_gen_bswap16_i64(swap, swap);
2934             break;
2935         case MO_32:
2936             tcg_gen_ext32u_i64(swap, val);
2937             tcg_gen_bswap32_i64(swap, swap);
2938             break;
2939         case MO_64:
2940             tcg_gen_bswap64_i64(swap, val);
2941             break;
2942         default:
2943             g_assert_not_reached();
2944         }
2945         val = swap;
2946         memop &= ~MO_BSWAP;
2947     }
2949     gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
2951     if (swap) {
2952         tcg_temp_free_i64(swap);
2953     }
2956 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2958     switch (opc & MO_SSIZE) {
2959     case MO_SB:
2960         tcg_gen_ext8s_i32(ret, val);
2961         break;
2962     case MO_UB:
2963         tcg_gen_ext8u_i32(ret, val);
2964         break;
2965     case MO_SW:
2966         tcg_gen_ext16s_i32(ret, val);
2967         break;
2968     case MO_UW:
2969         tcg_gen_ext16u_i32(ret, val);
2970         break;
2971     default:
2972         tcg_gen_mov_i32(ret, val);
2973         break;
2974     }
2977 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2979     switch (opc & MO_SSIZE) {
2980     case MO_SB:
2981         tcg_gen_ext8s_i64(ret, val);
2982         break;
2983     case MO_UB:
2984         tcg_gen_ext8u_i64(ret, val);
2985         break;
2986     case MO_SW:
2987         tcg_gen_ext16s_i64(ret, val);
2988         break;
2989     case MO_UW:
2990         tcg_gen_ext16u_i64(ret, val);
2991         break;
2992     case MO_SL:
2993         tcg_gen_ext32s_i64(ret, val);
2994         break;
2995     case MO_UL:
2996         tcg_gen_ext32u_i64(ret, val);
2997         break;
2998     default:
2999         tcg_gen_mov_i64(ret, val);
3000         break;
3001     }
3004 #ifdef CONFIG_SOFTMMU
3005 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3006                                   TCGv_i32, TCGv_i32, TCGv_i32);
3007 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3008                                   TCGv_i64, TCGv_i64, TCGv_i32);
3009 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3010                                   TCGv_i32, TCGv_i32);
3011 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3012                                   TCGv_i64, TCGv_i32);
3013 #else
3014 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
3015 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
3016 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
3017 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
3018 #endif
3020 #ifdef CONFIG_ATOMIC64
3021 # define WITH_ATOMIC64(X) X,
3022 #else
3023 # define WITH_ATOMIC64(X)
3024 #endif
3026 static void * const table_cmpxchg[16] = {
3027     [MO_8] = gen_helper_atomic_cmpxchgb,
3028     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3029     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3030     [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3031     [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
3032     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3033     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
3036 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3037                                 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
3039     memop = tcg_canonicalize_memop(memop, 0, 0);
3041     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3042         TCGv_i32 t1 = tcg_temp_new_i32();
3043         TCGv_i32 t2 = tcg_temp_new_i32();
3045         tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3047         tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3048         tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3049         tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3050         tcg_temp_free_i32(t2);
3052         if (memop & MO_SIGN) {
3053             tcg_gen_ext_i32(retv, t1, memop);
3054         } else {
3055             tcg_gen_mov_i32(retv, t1);
3056         }
3057         tcg_temp_free_i32(t1);
3058     } else {
3059         gen_atomic_cx_i32 gen;
3061         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3062         tcg_debug_assert(gen != NULL);
3064 #ifdef CONFIG_SOFTMMU
3065         {
3066             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3067             gen(retv, cpu_env, addr, cmpv, newv, oi);
3068             tcg_temp_free_i32(oi);
3069         }
3070 #else
3071         gen(retv, cpu_env, addr, cmpv, newv);
3072 #endif
3074         if (memop & MO_SIGN) {
3075             tcg_gen_ext_i32(retv, retv, memop);
3076         }
3077     }
3080 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3081                                 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
3083     memop = tcg_canonicalize_memop(memop, 1, 0);
3085     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3086         TCGv_i64 t1 = tcg_temp_new_i64();
3087         TCGv_i64 t2 = tcg_temp_new_i64();
3089         tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
3091         tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3092         tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3093         tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3094         tcg_temp_free_i64(t2);
3096         if (memop & MO_SIGN) {
3097             tcg_gen_ext_i64(retv, t1, memop);
3098         } else {
3099             tcg_gen_mov_i64(retv, t1);
3100         }
3101         tcg_temp_free_i64(t1);
3102     } else if ((memop & MO_SIZE) == MO_64) {
3103 #ifdef CONFIG_ATOMIC64
3104         gen_atomic_cx_i64 gen;
3106         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3107         tcg_debug_assert(gen != NULL);
3109 #ifdef CONFIG_SOFTMMU
3110         {
3111             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
3112             gen(retv, cpu_env, addr, cmpv, newv, oi);
3113             tcg_temp_free_i32(oi);
3114         }
3115 #else
3116         gen(retv, cpu_env, addr, cmpv, newv);
3117 #endif
3118 #else
3119         gen_helper_exit_atomic(cpu_env);
3120         /* Produce a result, so that we have a well-formed opcode stream
3121            with respect to uses of the result in the (dead) code following.  */
3122         tcg_gen_movi_i64(retv, 0);
3123 #endif /* CONFIG_ATOMIC64 */
3124     } else {
3125         TCGv_i32 c32 = tcg_temp_new_i32();
3126         TCGv_i32 n32 = tcg_temp_new_i32();
3127         TCGv_i32 r32 = tcg_temp_new_i32();
3129         tcg_gen_extrl_i64_i32(c32, cmpv);
3130         tcg_gen_extrl_i64_i32(n32, newv);
3131         tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3132         tcg_temp_free_i32(c32);
3133         tcg_temp_free_i32(n32);
3135         tcg_gen_extu_i32_i64(retv, r32);
3136         tcg_temp_free_i32(r32);
3138         if (memop & MO_SIGN) {
3139             tcg_gen_ext_i64(retv, retv, memop);
3140         }
3141     }
3144 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3145                                 TCGArg idx, TCGMemOp memop, bool new_val,
3146                                 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3148     TCGv_i32 t1 = tcg_temp_new_i32();
3149     TCGv_i32 t2 = tcg_temp_new_i32();
3151     memop = tcg_canonicalize_memop(memop, 0, 0);
3153     tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3154     gen(t2, t1, val);
3155     tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3157     tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3158     tcg_temp_free_i32(t1);
3159     tcg_temp_free_i32(t2);
3162 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3163                              TCGArg idx, TCGMemOp memop, void * const table[])
3165     gen_atomic_op_i32 gen;
3167     memop = tcg_canonicalize_memop(memop, 0, 0);
3169     gen = table[memop & (MO_SIZE | MO_BSWAP)];
3170     tcg_debug_assert(gen != NULL);
3172 #ifdef CONFIG_SOFTMMU
3173     {
3174         TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3175         gen(ret, cpu_env, addr, val, oi);
3176         tcg_temp_free_i32(oi);
3177     }
3178 #else
3179     gen(ret, cpu_env, addr, val);
3180 #endif
3182     if (memop & MO_SIGN) {
3183         tcg_gen_ext_i32(ret, ret, memop);
3184     }
3187 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3188                                 TCGArg idx, TCGMemOp memop, bool new_val,
3189                                 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3191     TCGv_i64 t1 = tcg_temp_new_i64();
3192     TCGv_i64 t2 = tcg_temp_new_i64();
3194     memop = tcg_canonicalize_memop(memop, 1, 0);
3196     tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3197     gen(t2, t1, val);
3198     tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3200     tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3201     tcg_temp_free_i64(t1);
3202     tcg_temp_free_i64(t2);
3205 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3206                              TCGArg idx, TCGMemOp memop, void * const table[])
3208     memop = tcg_canonicalize_memop(memop, 1, 0);
3210     if ((memop & MO_SIZE) == MO_64) {
3211 #ifdef CONFIG_ATOMIC64
3212         gen_atomic_op_i64 gen;
3214         gen = table[memop & (MO_SIZE | MO_BSWAP)];
3215         tcg_debug_assert(gen != NULL);
3217 #ifdef CONFIG_SOFTMMU
3218         {
3219             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3220             gen(ret, cpu_env, addr, val, oi);
3221             tcg_temp_free_i32(oi);
3222         }
3223 #else
3224         gen(ret, cpu_env, addr, val);
3225 #endif
3226 #else
3227         gen_helper_exit_atomic(cpu_env);
3228         /* Produce a result, so that we have a well-formed opcode stream
3229            with respect to uses of the result in the (dead) code following.  */
3230         tcg_gen_movi_i64(ret, 0);
3231 #endif /* CONFIG_ATOMIC64 */
3232     } else {
3233         TCGv_i32 v32 = tcg_temp_new_i32();
3234         TCGv_i32 r32 = tcg_temp_new_i32();
3236         tcg_gen_extrl_i64_i32(v32, val);
3237         do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3238         tcg_temp_free_i32(v32);
3240         tcg_gen_extu_i32_i64(ret, r32);
3241         tcg_temp_free_i32(r32);
3243         if (memop & MO_SIGN) {
3244             tcg_gen_ext_i64(ret, ret, memop);
3245         }
3246     }
3249 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
3250 static void * const table_##NAME[16] = {                                \
3251     [MO_8] = gen_helper_atomic_##NAME##b,                               \
3252     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
3253     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
3254     [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
3255     [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
3256     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le)     \
3257     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be)     \
3258 };                                                                      \
3259 void tcg_gen_atomic_##NAME##_i32                                        \
3260     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
3261 {                                                                       \
3262     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3263         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
3264     } else {                                                            \
3265         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
3266                             tcg_gen_##OP##_i32);                        \
3267     }                                                                   \
3268 }                                                                       \
3269 void tcg_gen_atomic_##NAME##_i64                                        \
3270     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
3271 {                                                                       \
3272     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3273         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
3274     } else {                                                            \
3275         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
3276                             tcg_gen_##OP##_i64);                        \
3277     }                                                                   \
3280 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3281 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3282 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3283 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3284 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3285 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3286 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3287 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3289 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3290 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3291 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3292 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3293 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3294 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3295 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3296 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3298 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3300     tcg_gen_mov_i32(r, b);
3303 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3305     tcg_gen_mov_i64(r, b);
3308 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3310 #undef GEN_ATOMIC_HELPER