?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'remotes/ericb/tags/pull-nbd-2018-03-01' into staging
[qemu.git] / tcg / tcg-op.c
blob34677873239397d64bda42716a2ddd1bd609d63c
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "qemu-common.h"
27 #include "cpu.h"
28 #include "exec/exec-all.h"
29 #include "tcg.h"
30 #include "tcg-op.h"
31 #include "tcg-mo.h"
32 #include "trace-tcg.h"
33 #include "trace/mem.h"
34
35 /* Reduce the number of ifdefs below. This assumes that all uses of
36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
37 the compiler can eliminate. */
38 #if TCG_TARGET_REG_BITS == 64
39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
41 #define TCGV_LOW TCGV_LOW_link_error
42 #define TCGV_HIGH TCGV_HIGH_link_error
43 #endif
44
45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
46 {
47 TCGOp *op = tcg_emit_op(opc);
48 op->args[0] = a1;
49 }
50
51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
52 {
53 TCGOp *op = tcg_emit_op(opc);
54 op->args[0] = a1;
55 op->args[1] = a2;
56 }
57
58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
59 {
60 TCGOp *op = tcg_emit_op(opc);
61 op->args[0] = a1;
62 op->args[1] = a2;
63 op->args[2] = a3;
64 }
65
66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
67 {
68 TCGOp *op = tcg_emit_op(opc);
69 op->args[0] = a1;
70 op->args[1] = a2;
71 op->args[2] = a3;
72 op->args[3] = a4;
73 }
74
75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76 TCGArg a4, TCGArg a5)
77 {
78 TCGOp *op = tcg_emit_op(opc);
79 op->args[0] = a1;
80 op->args[1] = a2;
81 op->args[2] = a3;
82 op->args[3] = a4;
83 op->args[4] = a5;
84 }
85
86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
87 TCGArg a4, TCGArg a5, TCGArg a6)
88 {
89 TCGOp *op = tcg_emit_op(opc);
90 op->args[0] = a1;
91 op->args[1] = a2;
92 op->args[2] = a3;
93 op->args[3] = a4;
94 op->args[4] = a5;
95 op->args[5] = a6;
96 }
97
98 void tcg_gen_mb(TCGBar mb_type)
99 {
100 if (tcg_ctx->tb_cflags & CF_PARALLEL) {
101 tcg_gen_op1(INDEX_op_mb, mb_type);
102 }
103 }
104
105 /* 32 bit ops */
106
107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
108 {
109 /* some cases can be optimized here */
110 if (arg2 == 0) {
111 tcg_gen_mov_i32(ret, arg1);
112 } else {
113 TCGv_i32 t0 = tcg_const_i32(arg2);
114 tcg_gen_add_i32(ret, arg1, t0);
115 tcg_temp_free_i32(t0);
116 }
117 }
118
119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120 {
121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
122 /* Don't recurse with tcg_gen_neg_i32. */
123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
124 } else {
125 TCGv_i32 t0 = tcg_const_i32(arg1);
126 tcg_gen_sub_i32(ret, t0, arg2);
127 tcg_temp_free_i32(t0);
128 }
129 }
130
131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
132 {
133 /* some cases can be optimized here */
134 if (arg2 == 0) {
135 tcg_gen_mov_i32(ret, arg1);
136 } else {
137 TCGv_i32 t0 = tcg_const_i32(arg2);
138 tcg_gen_sub_i32(ret, arg1, t0);
139 tcg_temp_free_i32(t0);
140 }
141 }
142
143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
144 {
145 TCGv_i32 t0;
146 /* Some cases can be optimized here. */
147 switch (arg2) {
148 case 0:
149 tcg_gen_movi_i32(ret, 0);
150 return;
151 case -1:
152 tcg_gen_mov_i32(ret, arg1);
153 return;
154 case 0xff:
155 /* Don't recurse with tcg_gen_ext8u_i32. */
156 if (TCG_TARGET_HAS_ext8u_i32) {
157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
158 return;
159 }
160 break;
161 case 0xffff:
162 if (TCG_TARGET_HAS_ext16u_i32) {
163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
164 return;
165 }
166 break;
167 }
168 t0 = tcg_const_i32(arg2);
169 tcg_gen_and_i32(ret, arg1, t0);
170 tcg_temp_free_i32(t0);
171 }
172
173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
174 {
175 /* Some cases can be optimized here. */
176 if (arg2 == -1) {
177 tcg_gen_movi_i32(ret, -1);
178 } else if (arg2 == 0) {
179 tcg_gen_mov_i32(ret, arg1);
180 } else {
181 TCGv_i32 t0 = tcg_const_i32(arg2);
182 tcg_gen_or_i32(ret, arg1, t0);
183 tcg_temp_free_i32(t0);
184 }
185 }
186
187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
188 {
189 /* Some cases can be optimized here. */
190 if (arg2 == 0) {
191 tcg_gen_mov_i32(ret, arg1);
192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
193 /* Don't recurse with tcg_gen_not_i32. */
194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
195 } else {
196 TCGv_i32 t0 = tcg_const_i32(arg2);
197 tcg_gen_xor_i32(ret, arg1, t0);
198 tcg_temp_free_i32(t0);
199 }
200 }
201
202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
203 {
204 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
205 if (arg2 == 0) {
206 tcg_gen_mov_i32(ret, arg1);
207 } else {
208 TCGv_i32 t0 = tcg_const_i32(arg2);
209 tcg_gen_shl_i32(ret, arg1, t0);
210 tcg_temp_free_i32(t0);
211 }
212 }
213
214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
215 {
216 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
217 if (arg2 == 0) {
218 tcg_gen_mov_i32(ret, arg1);
219 } else {
220 TCGv_i32 t0 = tcg_const_i32(arg2);
221 tcg_gen_shr_i32(ret, arg1, t0);
222 tcg_temp_free_i32(t0);
223 }
224 }
225
226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
227 {
228 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
229 if (arg2 == 0) {
230 tcg_gen_mov_i32(ret, arg1);
231 } else {
232 TCGv_i32 t0 = tcg_const_i32(arg2);
233 tcg_gen_sar_i32(ret, arg1, t0);
234 tcg_temp_free_i32(t0);
235 }
236 }
237
238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
239 {
240 if (cond == TCG_COND_ALWAYS) {
241 tcg_gen_br(l);
242 } else if (cond != TCG_COND_NEVER) {
243 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
244 }
245 }
246
247 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
248 {
249 if (cond == TCG_COND_ALWAYS) {
250 tcg_gen_br(l);
251 } else if (cond != TCG_COND_NEVER) {
252 TCGv_i32 t0 = tcg_const_i32(arg2);
253 tcg_gen_brcond_i32(cond, arg1, t0, l);
254 tcg_temp_free_i32(t0);
255 }
256 }
257
258 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
259 TCGv_i32 arg1, TCGv_i32 arg2)
260 {
261 if (cond == TCG_COND_ALWAYS) {
262 tcg_gen_movi_i32(ret, 1);
263 } else if (cond == TCG_COND_NEVER) {
264 tcg_gen_movi_i32(ret, 0);
265 } else {
266 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
267 }
268 }
269
270 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
271 TCGv_i32 arg1, int32_t arg2)
272 {
273 TCGv_i32 t0 = tcg_const_i32(arg2);
274 tcg_gen_setcond_i32(cond, ret, arg1, t0);
275 tcg_temp_free_i32(t0);
276 }
277
278 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
279 {
280 TCGv_i32 t0 = tcg_const_i32(arg2);
281 tcg_gen_mul_i32(ret, arg1, t0);
282 tcg_temp_free_i32(t0);
283 }
284
285 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
286 {
287 if (TCG_TARGET_HAS_div_i32) {
288 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
289 } else if (TCG_TARGET_HAS_div2_i32) {
290 TCGv_i32 t0 = tcg_temp_new_i32();
291 tcg_gen_sari_i32(t0, arg1, 31);
292 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
293 tcg_temp_free_i32(t0);
294 } else {
295 gen_helper_div_i32(ret, arg1, arg2);
296 }
297 }
298
299 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
300 {
301 if (TCG_TARGET_HAS_rem_i32) {
302 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
303 } else if (TCG_TARGET_HAS_div_i32) {
304 TCGv_i32 t0 = tcg_temp_new_i32();
305 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
306 tcg_gen_mul_i32(t0, t0, arg2);
307 tcg_gen_sub_i32(ret, arg1, t0);
308 tcg_temp_free_i32(t0);
309 } else if (TCG_TARGET_HAS_div2_i32) {
310 TCGv_i32 t0 = tcg_temp_new_i32();
311 tcg_gen_sari_i32(t0, arg1, 31);
312 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
313 tcg_temp_free_i32(t0);
314 } else {
315 gen_helper_rem_i32(ret, arg1, arg2);
316 }
317 }
318
319 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
320 {
321 if (TCG_TARGET_HAS_div_i32) {
322 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
323 } else if (TCG_TARGET_HAS_div2_i32) {
324 TCGv_i32 t0 = tcg_temp_new_i32();
325 tcg_gen_movi_i32(t0, 0);
326 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
327 tcg_temp_free_i32(t0);
328 } else {
329 gen_helper_divu_i32(ret, arg1, arg2);
330 }
331 }
332
333 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
334 {
335 if (TCG_TARGET_HAS_rem_i32) {
336 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
337 } else if (TCG_TARGET_HAS_div_i32) {
338 TCGv_i32 t0 = tcg_temp_new_i32();
339 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
340 tcg_gen_mul_i32(t0, t0, arg2);
341 tcg_gen_sub_i32(ret, arg1, t0);
342 tcg_temp_free_i32(t0);
343 } else if (TCG_TARGET_HAS_div2_i32) {
344 TCGv_i32 t0 = tcg_temp_new_i32();
345 tcg_gen_movi_i32(t0, 0);
346 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
347 tcg_temp_free_i32(t0);
348 } else {
349 gen_helper_remu_i32(ret, arg1, arg2);
350 }
351 }
352
353 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
354 {
355 if (TCG_TARGET_HAS_andc_i32) {
356 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
357 } else {
358 TCGv_i32 t0 = tcg_temp_new_i32();
359 tcg_gen_not_i32(t0, arg2);
360 tcg_gen_and_i32(ret, arg1, t0);
361 tcg_temp_free_i32(t0);
362 }
363 }
364
365 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
366 {
367 if (TCG_TARGET_HAS_eqv_i32) {
368 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
369 } else {
370 tcg_gen_xor_i32(ret, arg1, arg2);
371 tcg_gen_not_i32(ret, ret);
372 }
373 }
374
375 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
376 {
377 if (TCG_TARGET_HAS_nand_i32) {
378 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
379 } else {
380 tcg_gen_and_i32(ret, arg1, arg2);
381 tcg_gen_not_i32(ret, ret);
382 }
383 }
384
385 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
386 {
387 if (TCG_TARGET_HAS_nor_i32) {
388 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
389 } else {
390 tcg_gen_or_i32(ret, arg1, arg2);
391 tcg_gen_not_i32(ret, ret);
392 }
393 }
394
395 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
396 {
397 if (TCG_TARGET_HAS_orc_i32) {
398 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
399 } else {
400 TCGv_i32 t0 = tcg_temp_new_i32();
401 tcg_gen_not_i32(t0, arg2);
402 tcg_gen_or_i32(ret, arg1, t0);
403 tcg_temp_free_i32(t0);
404 }
405 }
406
407 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
408 {
409 if (TCG_TARGET_HAS_clz_i32) {
410 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
411 } else if (TCG_TARGET_HAS_clz_i64) {
412 TCGv_i64 t1 = tcg_temp_new_i64();
413 TCGv_i64 t2 = tcg_temp_new_i64();
414 tcg_gen_extu_i32_i64(t1, arg1);
415 tcg_gen_extu_i32_i64(t2, arg2);
416 tcg_gen_addi_i64(t2, t2, 32);
417 tcg_gen_clz_i64(t1, t1, t2);
418 tcg_gen_extrl_i64_i32(ret, t1);
419 tcg_temp_free_i64(t1);
420 tcg_temp_free_i64(t2);
421 tcg_gen_subi_i32(ret, ret, 32);
422 } else {
423 gen_helper_clz_i32(ret, arg1, arg2);
424 }
425 }
426
427 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
428 {
429 TCGv_i32 t = tcg_const_i32(arg2);
430 tcg_gen_clz_i32(ret, arg1, t);
431 tcg_temp_free_i32(t);
432 }
433
434 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
435 {
436 if (TCG_TARGET_HAS_ctz_i32) {
437 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
438 } else if (TCG_TARGET_HAS_ctz_i64) {
439 TCGv_i64 t1 = tcg_temp_new_i64();
440 TCGv_i64 t2 = tcg_temp_new_i64();
441 tcg_gen_extu_i32_i64(t1, arg1);
442 tcg_gen_extu_i32_i64(t2, arg2);
443 tcg_gen_ctz_i64(t1, t1, t2);
444 tcg_gen_extrl_i64_i32(ret, t1);
445 tcg_temp_free_i64(t1);
446 tcg_temp_free_i64(t2);
447 } else if (TCG_TARGET_HAS_ctpop_i32
448 || TCG_TARGET_HAS_ctpop_i64
449 || TCG_TARGET_HAS_clz_i32
450 || TCG_TARGET_HAS_clz_i64) {
451 TCGv_i32 z, t = tcg_temp_new_i32();
452
453 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
454 tcg_gen_subi_i32(t, arg1, 1);
455 tcg_gen_andc_i32(t, t, arg1);
456 tcg_gen_ctpop_i32(t, t);
457 } else {
458 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
459 tcg_gen_neg_i32(t, arg1);
460 tcg_gen_and_i32(t, t, arg1);
461 tcg_gen_clzi_i32(t, t, 32);
462 tcg_gen_xori_i32(t, t, 31);
463 }
464 z = tcg_const_i32(0);
465 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
466 tcg_temp_free_i32(t);
467 tcg_temp_free_i32(z);
468 } else {
469 gen_helper_ctz_i32(ret, arg1, arg2);
470 }
471 }
472
473 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
474 {
475 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
476 /* This equivalence has the advantage of not requiring a fixup. */
477 TCGv_i32 t = tcg_temp_new_i32();
478 tcg_gen_subi_i32(t, arg1, 1);
479 tcg_gen_andc_i32(t, t, arg1);
480 tcg_gen_ctpop_i32(ret, t);
481 tcg_temp_free_i32(t);
482 } else {
483 TCGv_i32 t = tcg_const_i32(arg2);
484 tcg_gen_ctz_i32(ret, arg1, t);
485 tcg_temp_free_i32(t);
486 }
487 }
488
489 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
490 {
491 if (TCG_TARGET_HAS_clz_i32) {
492 TCGv_i32 t = tcg_temp_new_i32();
493 tcg_gen_sari_i32(t, arg, 31);
494 tcg_gen_xor_i32(t, t, arg);
495 tcg_gen_clzi_i32(t, t, 32);
496 tcg_gen_subi_i32(ret, t, 1);
497 tcg_temp_free_i32(t);
498 } else {
499 gen_helper_clrsb_i32(ret, arg);
500 }
501 }
502
503 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
504 {
505 if (TCG_TARGET_HAS_ctpop_i32) {
506 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
507 } else if (TCG_TARGET_HAS_ctpop_i64) {
508 TCGv_i64 t = tcg_temp_new_i64();
509 tcg_gen_extu_i32_i64(t, arg1);
510 tcg_gen_ctpop_i64(t, t);
511 tcg_gen_extrl_i64_i32(ret, t);
512 tcg_temp_free_i64(t);
513 } else {
514 gen_helper_ctpop_i32(ret, arg1);
515 }
516 }
517
518 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
519 {
520 if (TCG_TARGET_HAS_rot_i32) {
521 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
522 } else {
523 TCGv_i32 t0, t1;
524
525 t0 = tcg_temp_new_i32();
526 t1 = tcg_temp_new_i32();
527 tcg_gen_shl_i32(t0, arg1, arg2);
528 tcg_gen_subfi_i32(t1, 32, arg2);
529 tcg_gen_shr_i32(t1, arg1, t1);
530 tcg_gen_or_i32(ret, t0, t1);
531 tcg_temp_free_i32(t0);
532 tcg_temp_free_i32(t1);
533 }
534 }
535
536 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
537 {
538 tcg_debug_assert(arg2 < 32);
539 /* some cases can be optimized here */
540 if (arg2 == 0) {
541 tcg_gen_mov_i32(ret, arg1);
542 } else if (TCG_TARGET_HAS_rot_i32) {
543 TCGv_i32 t0 = tcg_const_i32(arg2);
544 tcg_gen_rotl_i32(ret, arg1, t0);
545 tcg_temp_free_i32(t0);
546 } else {
547 TCGv_i32 t0, t1;
548 t0 = tcg_temp_new_i32();
549 t1 = tcg_temp_new_i32();
550 tcg_gen_shli_i32(t0, arg1, arg2);
551 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
552 tcg_gen_or_i32(ret, t0, t1);
553 tcg_temp_free_i32(t0);
554 tcg_temp_free_i32(t1);
555 }
556 }
557
558 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
559 {
560 if (TCG_TARGET_HAS_rot_i32) {
561 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
562 } else {
563 TCGv_i32 t0, t1;
564
565 t0 = tcg_temp_new_i32();
566 t1 = tcg_temp_new_i32();
567 tcg_gen_shr_i32(t0, arg1, arg2);
568 tcg_gen_subfi_i32(t1, 32, arg2);
569 tcg_gen_shl_i32(t1, arg1, t1);
570 tcg_gen_or_i32(ret, t0, t1);
571 tcg_temp_free_i32(t0);
572 tcg_temp_free_i32(t1);
573 }
574 }
575
576 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
577 {
578 tcg_debug_assert(arg2 < 32);
579 /* some cases can be optimized here */
580 if (arg2 == 0) {
581 tcg_gen_mov_i32(ret, arg1);
582 } else {
583 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
584 }
585 }
586
587 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
588 unsigned int ofs, unsigned int len)
589 {
590 uint32_t mask;
591 TCGv_i32 t1;
592
593 tcg_debug_assert(ofs < 32);
594 tcg_debug_assert(len > 0);
595 tcg_debug_assert(len <= 32);
596 tcg_debug_assert(ofs + len <= 32);
597
598 if (len == 32) {
599 tcg_gen_mov_i32(ret, arg2);
600 return;
601 }
602 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
603 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
604 return;
605 }
606
607 mask = (1u << len) - 1;
608 t1 = tcg_temp_new_i32();
609
610 if (ofs + len < 32) {
611 tcg_gen_andi_i32(t1, arg2, mask);
612 tcg_gen_shli_i32(t1, t1, ofs);
613 } else {
614 tcg_gen_shli_i32(t1, arg2, ofs);
615 }
616 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
617 tcg_gen_or_i32(ret, ret, t1);
618
619 tcg_temp_free_i32(t1);
620 }
621
622 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
623 unsigned int ofs, unsigned int len)
624 {
625 tcg_debug_assert(ofs < 32);
626 tcg_debug_assert(len > 0);
627 tcg_debug_assert(len <= 32);
628 tcg_debug_assert(ofs + len <= 32);
629
630 if (ofs + len == 32) {
631 tcg_gen_shli_i32(ret, arg, ofs);
632 } else if (ofs == 0) {
633 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
634 } else if (TCG_TARGET_HAS_deposit_i32
635 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
636 TCGv_i32 zero = tcg_const_i32(0);
637 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
638 tcg_temp_free_i32(zero);
639 } else {
640 /* To help two-operand hosts we prefer to zero-extend first,
641 which allows ARG to stay live. */
642 switch (len) {
643 case 16:
644 if (TCG_TARGET_HAS_ext16u_i32) {
645 tcg_gen_ext16u_i32(ret, arg);
646 tcg_gen_shli_i32(ret, ret, ofs);
647 return;
648 }
649 break;
650 case 8:
651 if (TCG_TARGET_HAS_ext8u_i32) {
652 tcg_gen_ext8u_i32(ret, arg);
653 tcg_gen_shli_i32(ret, ret, ofs);
654 return;
655 }
656 break;
657 }
658 /* Otherwise prefer zero-extension over AND for code size. */
659 switch (ofs + len) {
660 case 16:
661 if (TCG_TARGET_HAS_ext16u_i32) {
662 tcg_gen_shli_i32(ret, arg, ofs);
663 tcg_gen_ext16u_i32(ret, ret);
664 return;
665 }
666 break;
667 case 8:
668 if (TCG_TARGET_HAS_ext8u_i32) {
669 tcg_gen_shli_i32(ret, arg, ofs);
670 tcg_gen_ext8u_i32(ret, ret);
671 return;
672 }
673 break;
674 }
675 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
676 tcg_gen_shli_i32(ret, ret, ofs);
677 }
678 }
679
680 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
681 unsigned int ofs, unsigned int len)
682 {
683 tcg_debug_assert(ofs < 32);
684 tcg_debug_assert(len > 0);
685 tcg_debug_assert(len <= 32);
686 tcg_debug_assert(ofs + len <= 32);
687
688 /* Canonicalize certain special cases, even if extract is supported. */
689 if (ofs + len == 32) {
690 tcg_gen_shri_i32(ret, arg, 32 - len);
691 return;
692 }
693 if (ofs == 0) {
694 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
695 return;
696 }
697
698 if (TCG_TARGET_HAS_extract_i32
699 && TCG_TARGET_extract_i32_valid(ofs, len)) {
700 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
701 return;
702 }
703
704 /* Assume that zero-extension, if available, is cheaper than a shift. */
705 switch (ofs + len) {
706 case 16:
707 if (TCG_TARGET_HAS_ext16u_i32) {
708 tcg_gen_ext16u_i32(ret, arg);
709 tcg_gen_shri_i32(ret, ret, ofs);
710 return;
711 }
712 break;
713 case 8:
714 if (TCG_TARGET_HAS_ext8u_i32) {
715 tcg_gen_ext8u_i32(ret, arg);
716 tcg_gen_shri_i32(ret, ret, ofs);
717 return;
718 }
719 break;
720 }
721
722 /* ??? Ideally we'd know what values are available for immediate AND.
723 Assume that 8 bits are available, plus the special case of 16,
724 so that we get ext8u, ext16u. */
725 switch (len) {
726 case 1 ... 8: case 16:
727 tcg_gen_shri_i32(ret, arg, ofs);
728 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
729 break;
730 default:
731 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
732 tcg_gen_shri_i32(ret, ret, 32 - len);
733 break;
734 }
735 }
736
737 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
738 unsigned int ofs, unsigned int len)
739 {
740 tcg_debug_assert(ofs < 32);
741 tcg_debug_assert(len > 0);
742 tcg_debug_assert(len <= 32);
743 tcg_debug_assert(ofs + len <= 32);
744
745 /* Canonicalize certain special cases, even if extract is supported. */
746 if (ofs + len == 32) {
747 tcg_gen_sari_i32(ret, arg, 32 - len);
748 return;
749 }
750 if (ofs == 0) {
751 switch (len) {
752 case 16:
753 tcg_gen_ext16s_i32(ret, arg);
754 return;
755 case 8:
756 tcg_gen_ext8s_i32(ret, arg);
757 return;
758 }
759 }
760
761 if (TCG_TARGET_HAS_sextract_i32
762 && TCG_TARGET_extract_i32_valid(ofs, len)) {
763 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
764 return;
765 }
766
767 /* Assume that sign-extension, if available, is cheaper than a shift. */
768 switch (ofs + len) {
769 case 16:
770 if (TCG_TARGET_HAS_ext16s_i32) {
771 tcg_gen_ext16s_i32(ret, arg);
772 tcg_gen_sari_i32(ret, ret, ofs);
773 return;
774 }
775 break;
776 case 8:
777 if (TCG_TARGET_HAS_ext8s_i32) {
778 tcg_gen_ext8s_i32(ret, arg);
779 tcg_gen_sari_i32(ret, ret, ofs);
780 return;
781 }
782 break;
783 }
784 switch (len) {
785 case 16:
786 if (TCG_TARGET_HAS_ext16s_i32) {
787 tcg_gen_shri_i32(ret, arg, ofs);
788 tcg_gen_ext16s_i32(ret, ret);
789 return;
790 }
791 break;
792 case 8:
793 if (TCG_TARGET_HAS_ext8s_i32) {
794 tcg_gen_shri_i32(ret, arg, ofs);
795 tcg_gen_ext8s_i32(ret, ret);
796 return;
797 }
798 break;
799 }
800
801 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
802 tcg_gen_sari_i32(ret, ret, 32 - len);
803 }
804
805 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
806 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
807 {
808 if (cond == TCG_COND_ALWAYS) {
809 tcg_gen_mov_i32(ret, v1);
810 } else if (cond == TCG_COND_NEVER) {
811 tcg_gen_mov_i32(ret, v2);
812 } else if (TCG_TARGET_HAS_movcond_i32) {
813 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
814 } else {
815 TCGv_i32 t0 = tcg_temp_new_i32();
816 TCGv_i32 t1 = tcg_temp_new_i32();
817 tcg_gen_setcond_i32(cond, t0, c1, c2);
818 tcg_gen_neg_i32(t0, t0);
819 tcg_gen_and_i32(t1, v1, t0);
820 tcg_gen_andc_i32(ret, v2, t0);
821 tcg_gen_or_i32(ret, ret, t1);
822 tcg_temp_free_i32(t0);
823 tcg_temp_free_i32(t1);
824 }
825 }
826
827 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
828 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
829 {
830 if (TCG_TARGET_HAS_add2_i32) {
831 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
832 } else {
833 TCGv_i64 t0 = tcg_temp_new_i64();
834 TCGv_i64 t1 = tcg_temp_new_i64();
835 tcg_gen_concat_i32_i64(t0, al, ah);
836 tcg_gen_concat_i32_i64(t1, bl, bh);
837 tcg_gen_add_i64(t0, t0, t1);
838 tcg_gen_extr_i64_i32(rl, rh, t0);
839 tcg_temp_free_i64(t0);
840 tcg_temp_free_i64(t1);
841 }
842 }
843
844 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
845 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
846 {
847 if (TCG_TARGET_HAS_sub2_i32) {
848 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
849 } else {
850 TCGv_i64 t0 = tcg_temp_new_i64();
851 TCGv_i64 t1 = tcg_temp_new_i64();
852 tcg_gen_concat_i32_i64(t0, al, ah);
853 tcg_gen_concat_i32_i64(t1, bl, bh);
854 tcg_gen_sub_i64(t0, t0, t1);
855 tcg_gen_extr_i64_i32(rl, rh, t0);
856 tcg_temp_free_i64(t0);
857 tcg_temp_free_i64(t1);
858 }
859 }
860
861 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
862 {
863 if (TCG_TARGET_HAS_mulu2_i32) {
864 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
865 } else if (TCG_TARGET_HAS_muluh_i32) {
866 TCGv_i32 t = tcg_temp_new_i32();
867 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
868 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
869 tcg_gen_mov_i32(rl, t);
870 tcg_temp_free_i32(t);
871 } else {
872 TCGv_i64 t0 = tcg_temp_new_i64();
873 TCGv_i64 t1 = tcg_temp_new_i64();
874 tcg_gen_extu_i32_i64(t0, arg1);
875 tcg_gen_extu_i32_i64(t1, arg2);
876 tcg_gen_mul_i64(t0, t0, t1);
877 tcg_gen_extr_i64_i32(rl, rh, t0);
878 tcg_temp_free_i64(t0);
879 tcg_temp_free_i64(t1);
880 }
881 }
882
883 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
884 {
885 if (TCG_TARGET_HAS_muls2_i32) {
886 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
887 } else if (TCG_TARGET_HAS_mulsh_i32) {
888 TCGv_i32 t = tcg_temp_new_i32();
889 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
890 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
891 tcg_gen_mov_i32(rl, t);
892 tcg_temp_free_i32(t);
893 } else if (TCG_TARGET_REG_BITS == 32) {
894 TCGv_i32 t0 = tcg_temp_new_i32();
895 TCGv_i32 t1 = tcg_temp_new_i32();
896 TCGv_i32 t2 = tcg_temp_new_i32();
897 TCGv_i32 t3 = tcg_temp_new_i32();
898 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
899 /* Adjust for negative inputs. */
900 tcg_gen_sari_i32(t2, arg1, 31);
901 tcg_gen_sari_i32(t3, arg2, 31);
902 tcg_gen_and_i32(t2, t2, arg2);
903 tcg_gen_and_i32(t3, t3, arg1);
904 tcg_gen_sub_i32(rh, t1, t2);
905 tcg_gen_sub_i32(rh, rh, t3);
906 tcg_gen_mov_i32(rl, t0);
907 tcg_temp_free_i32(t0);
908 tcg_temp_free_i32(t1);
909 tcg_temp_free_i32(t2);
910 tcg_temp_free_i32(t3);
911 } else {
912 TCGv_i64 t0 = tcg_temp_new_i64();
913 TCGv_i64 t1 = tcg_temp_new_i64();
914 tcg_gen_ext_i32_i64(t0, arg1);
915 tcg_gen_ext_i32_i64(t1, arg2);
916 tcg_gen_mul_i64(t0, t0, t1);
917 tcg_gen_extr_i64_i32(rl, rh, t0);
918 tcg_temp_free_i64(t0);
919 tcg_temp_free_i64(t1);
920 }
921 }
922
923 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
924 {
925 if (TCG_TARGET_REG_BITS == 32) {
926 TCGv_i32 t0 = tcg_temp_new_i32();
927 TCGv_i32 t1 = tcg_temp_new_i32();
928 TCGv_i32 t2 = tcg_temp_new_i32();
929 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
930 /* Adjust for negative input for the signed arg1. */
931 tcg_gen_sari_i32(t2, arg1, 31);
932 tcg_gen_and_i32(t2, t2, arg2);
933 tcg_gen_sub_i32(rh, t1, t2);
934 tcg_gen_mov_i32(rl, t0);
935 tcg_temp_free_i32(t0);
936 tcg_temp_free_i32(t1);
937 tcg_temp_free_i32(t2);
938 } else {
939 TCGv_i64 t0 = tcg_temp_new_i64();
940 TCGv_i64 t1 = tcg_temp_new_i64();
941 tcg_gen_ext_i32_i64(t0, arg1);
942 tcg_gen_extu_i32_i64(t1, arg2);
943 tcg_gen_mul_i64(t0, t0, t1);
944 tcg_gen_extr_i64_i32(rl, rh, t0);
945 tcg_temp_free_i64(t0);
946 tcg_temp_free_i64(t1);
947 }
948 }
949
950 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
951 {
952 if (TCG_TARGET_HAS_ext8s_i32) {
953 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
954 } else {
955 tcg_gen_shli_i32(ret, arg, 24);
956 tcg_gen_sari_i32(ret, ret, 24);
957 }
958 }
959
960 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
961 {
962 if (TCG_TARGET_HAS_ext16s_i32) {
963 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
964 } else {
965 tcg_gen_shli_i32(ret, arg, 16);
966 tcg_gen_sari_i32(ret, ret, 16);
967 }
968 }
969
970 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
971 {
972 if (TCG_TARGET_HAS_ext8u_i32) {
973 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
974 } else {
975 tcg_gen_andi_i32(ret, arg, 0xffu);
976 }
977 }
978
979 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
980 {
981 if (TCG_TARGET_HAS_ext16u_i32) {
982 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
983 } else {
984 tcg_gen_andi_i32(ret, arg, 0xffffu);
985 }
986 }
987
988 /* Note: we assume the two high bytes are set to zero */
989 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
990 {
991 if (TCG_TARGET_HAS_bswap16_i32) {
992 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
993 } else {
994 TCGv_i32 t0 = tcg_temp_new_i32();
995
996 tcg_gen_ext8u_i32(t0, arg);
997 tcg_gen_shli_i32(t0, t0, 8);
998 tcg_gen_shri_i32(ret, arg, 8);
999 tcg_gen_or_i32(ret, ret, t0);
1000 tcg_temp_free_i32(t0);
1001 }
1002 }
1003
1004 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1005 {
1006 if (TCG_TARGET_HAS_bswap32_i32) {
1007 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1008 } else {
1009 TCGv_i32 t0, t1;
1010 t0 = tcg_temp_new_i32();
1011 t1 = tcg_temp_new_i32();
1012
1013 tcg_gen_shli_i32(t0, arg, 24);
1014
1015 tcg_gen_andi_i32(t1, arg, 0x0000ff00);
1016 tcg_gen_shli_i32(t1, t1, 8);
1017 tcg_gen_or_i32(t0, t0, t1);
1018
1019 tcg_gen_shri_i32(t1, arg, 8);
1020 tcg_gen_andi_i32(t1, t1, 0x0000ff00);
1021 tcg_gen_or_i32(t0, t0, t1);
1022
1023 tcg_gen_shri_i32(t1, arg, 24);
1024 tcg_gen_or_i32(ret, t0, t1);
1025 tcg_temp_free_i32(t0);
1026 tcg_temp_free_i32(t1);
1027 }
1028 }
1029
1030 /* 64-bit ops */
1031
1032 #if TCG_TARGET_REG_BITS == 32
1033 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1034
1035 void tcg_gen_discard_i64(TCGv_i64 arg)
1036 {
1037 tcg_gen_discard_i32(TCGV_LOW(arg));
1038 tcg_gen_discard_i32(TCGV_HIGH(arg));
1039 }
1040
1041 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1042 {
1043 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1044 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1045 }
1046
1047 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1048 {
1049 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1050 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1051 }
1052
1053 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1054 {
1055 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1056 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1057 }
1058
1059 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1060 {
1061 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1062 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1063 }
1064
1065 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1066 {
1067 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1068 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1069 }
1070
1071 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1072 {
1073 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1074 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1075 }
1076
1077 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1078 {
1079 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1080 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1081 }
1082
1083 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1084 {
1085 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1086 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1087 }
1088
1089 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1090 {
1091 /* Since arg2 and ret have different types,
1092 they cannot be the same temporary */
1093 #ifdef HOST_WORDS_BIGENDIAN
1094 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1095 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1096 #else
1097 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1098 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1099 #endif
1100 }
1101
1102 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1103 {
1104 #ifdef HOST_WORDS_BIGENDIAN
1105 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1106 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1107 #else
1108 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1109 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1110 #endif
1111 }
1112
1113 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1114 {
1115 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1116 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1117 }
1118
1119 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1120 {
1121 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1122 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1123 }
1124
1125 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1126 {
1127 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1128 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1129 }
1130
1131 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1132 {
1133 gen_helper_shl_i64(ret, arg1, arg2);
1134 }
1135
1136 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1137 {
1138 gen_helper_shr_i64(ret, arg1, arg2);
1139 }
1140
1141 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1142 {
1143 gen_helper_sar_i64(ret, arg1, arg2);
1144 }
1145
1146 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1147 {
1148 TCGv_i64 t0;
1149 TCGv_i32 t1;
1150
1151 t0 = tcg_temp_new_i64();
1152 t1 = tcg_temp_new_i32();
1153
1154 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1155 TCGV_LOW(arg1), TCGV_LOW(arg2));
1156
1157 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1158 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1159 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1160 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1161
1162 tcg_gen_mov_i64(ret, t0);
1163 tcg_temp_free_i64(t0);
1164 tcg_temp_free_i32(t1);
1165 }
1166 #endif /* TCG_TARGET_REG_SIZE == 32 */
1167
1168 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1169 {
1170 /* some cases can be optimized here */
1171 if (arg2 == 0) {
1172 tcg_gen_mov_i64(ret, arg1);
1173 } else {
1174 TCGv_i64 t0 = tcg_const_i64(arg2);
1175 tcg_gen_add_i64(ret, arg1, t0);
1176 tcg_temp_free_i64(t0);
1177 }
1178 }
1179
1180 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1181 {
1182 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1183 /* Don't recurse with tcg_gen_neg_i64. */
1184 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1185 } else {
1186 TCGv_i64 t0 = tcg_const_i64(arg1);
1187 tcg_gen_sub_i64(ret, t0, arg2);
1188 tcg_temp_free_i64(t0);
1189 }
1190 }
1191
1192 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1193 {
1194 /* some cases can be optimized here */
1195 if (arg2 == 0) {
1196 tcg_gen_mov_i64(ret, arg1);
1197 } else {
1198 TCGv_i64 t0 = tcg_const_i64(arg2);
1199 tcg_gen_sub_i64(ret, arg1, t0);
1200 tcg_temp_free_i64(t0);
1201 }
1202 }
1203
1204 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1205 {
1206 TCGv_i64 t0;
1207
1208 if (TCG_TARGET_REG_BITS == 32) {
1209 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1210 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1211 return;
1212 }
1213
1214 /* Some cases can be optimized here. */
1215 switch (arg2) {
1216 case 0:
1217 tcg_gen_movi_i64(ret, 0);
1218 return;
1219 case -1:
1220 tcg_gen_mov_i64(ret, arg1);
1221 return;
1222 case 0xff:
1223 /* Don't recurse with tcg_gen_ext8u_i64. */
1224 if (TCG_TARGET_HAS_ext8u_i64) {
1225 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1226 return;
1227 }
1228 break;
1229 case 0xffff:
1230 if (TCG_TARGET_HAS_ext16u_i64) {
1231 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1232 return;
1233 }
1234 break;
1235 case 0xffffffffu:
1236 if (TCG_TARGET_HAS_ext32u_i64) {
1237 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1238 return;
1239 }
1240 break;
1241 }
1242 t0 = tcg_const_i64(arg2);
1243 tcg_gen_and_i64(ret, arg1, t0);
1244 tcg_temp_free_i64(t0);
1245 }
1246
1247 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1248 {
1249 if (TCG_TARGET_REG_BITS == 32) {
1250 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1251 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1252 return;
1253 }
1254 /* Some cases can be optimized here. */
1255 if (arg2 == -1) {
1256 tcg_gen_movi_i64(ret, -1);
1257 } else if (arg2 == 0) {
1258 tcg_gen_mov_i64(ret, arg1);
1259 } else {
1260 TCGv_i64 t0 = tcg_const_i64(arg2);
1261 tcg_gen_or_i64(ret, arg1, t0);
1262 tcg_temp_free_i64(t0);
1263 }
1264 }
1265
1266 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1267 {
1268 if (TCG_TARGET_REG_BITS == 32) {
1269 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1270 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1271 return;
1272 }
1273 /* Some cases can be optimized here. */
1274 if (arg2 == 0) {
1275 tcg_gen_mov_i64(ret, arg1);
1276 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1277 /* Don't recurse with tcg_gen_not_i64. */
1278 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1279 } else {
1280 TCGv_i64 t0 = tcg_const_i64(arg2);
1281 tcg_gen_xor_i64(ret, arg1, t0);
1282 tcg_temp_free_i64(t0);
1283 }
1284 }
1285
1286 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1287 unsigned c, bool right, bool arith)
1288 {
1289 tcg_debug_assert(c < 64);
1290 if (c == 0) {
1291 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1292 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1293 } else if (c >= 32) {
1294 c -= 32;
1295 if (right) {
1296 if (arith) {
1297 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1298 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1299 } else {
1300 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1301 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1302 }
1303 } else {
1304 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1305 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1306 }
1307 } else {
1308 TCGv_i32 t0, t1;
1309
1310 t0 = tcg_temp_new_i32();
1311 t1 = tcg_temp_new_i32();
1312 if (right) {
1313 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1314 if (arith) {
1315 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1316 } else {
1317 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1318 }
1319 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1320 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1321 tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1322 } else {
1323 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1324 /* Note: ret can be the same as arg1, so we use t1 */
1325 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1326 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1327 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1328 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1329 }
1330 tcg_temp_free_i32(t0);
1331 tcg_temp_free_i32(t1);
1332 }
1333 }
1334
1335 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1336 {
1337 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1338 if (TCG_TARGET_REG_BITS == 32) {
1339 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1340 } else if (arg2 == 0) {
1341 tcg_gen_mov_i64(ret, arg1);
1342 } else {
1343 TCGv_i64 t0 = tcg_const_i64(arg2);
1344 tcg_gen_shl_i64(ret, arg1, t0);
1345 tcg_temp_free_i64(t0);
1346 }
1347 }
1348
1349 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1350 {
1351 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1352 if (TCG_TARGET_REG_BITS == 32) {
1353 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1354 } else if (arg2 == 0) {
1355 tcg_gen_mov_i64(ret, arg1);
1356 } else {
1357 TCGv_i64 t0 = tcg_const_i64(arg2);
1358 tcg_gen_shr_i64(ret, arg1, t0);
1359 tcg_temp_free_i64(t0);
1360 }
1361 }
1362
1363 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1364 {
1365 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1366 if (TCG_TARGET_REG_BITS == 32) {
1367 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1368 } else if (arg2 == 0) {
1369 tcg_gen_mov_i64(ret, arg1);
1370 } else {
1371 TCGv_i64 t0 = tcg_const_i64(arg2);
1372 tcg_gen_sar_i64(ret, arg1, t0);
1373 tcg_temp_free_i64(t0);
1374 }
1375 }
1376
1377 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1378 {
1379 if (cond == TCG_COND_ALWAYS) {
1380 tcg_gen_br(l);
1381 } else if (cond != TCG_COND_NEVER) {
1382 if (TCG_TARGET_REG_BITS == 32) {
1383 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1384 TCGV_HIGH(arg1), TCGV_LOW(arg2),
1385 TCGV_HIGH(arg2), cond, label_arg(l));
1386 } else {
1387 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1388 label_arg(l));
1389 }
1390 }
1391 }
1392
1393 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1394 {
1395 if (cond == TCG_COND_ALWAYS) {
1396 tcg_gen_br(l);
1397 } else if (cond != TCG_COND_NEVER) {
1398 TCGv_i64 t0 = tcg_const_i64(arg2);
1399 tcg_gen_brcond_i64(cond, arg1, t0, l);
1400 tcg_temp_free_i64(t0);
1401 }
1402 }
1403
1404 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1405 TCGv_i64 arg1, TCGv_i64 arg2)
1406 {
1407 if (cond == TCG_COND_ALWAYS) {
1408 tcg_gen_movi_i64(ret, 1);
1409 } else if (cond == TCG_COND_NEVER) {
1410 tcg_gen_movi_i64(ret, 0);
1411 } else {
1412 if (TCG_TARGET_REG_BITS == 32) {
1413 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1414 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1415 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1416 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1417 } else {
1418 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1419 }
1420 }
1421 }
1422
1423 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1424 TCGv_i64 arg1, int64_t arg2)
1425 {
1426 TCGv_i64 t0 = tcg_const_i64(arg2);
1427 tcg_gen_setcond_i64(cond, ret, arg1, t0);
1428 tcg_temp_free_i64(t0);
1429 }
1430
1431 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1432 {
1433 TCGv_i64 t0 = tcg_const_i64(arg2);
1434 tcg_gen_mul_i64(ret, arg1, t0);
1435 tcg_temp_free_i64(t0);
1436 }
1437
1438 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1439 {
1440 if (TCG_TARGET_HAS_div_i64) {
1441 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1442 } else if (TCG_TARGET_HAS_div2_i64) {
1443 TCGv_i64 t0 = tcg_temp_new_i64();
1444 tcg_gen_sari_i64(t0, arg1, 63);
1445 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1446 tcg_temp_free_i64(t0);
1447 } else {
1448 gen_helper_div_i64(ret, arg1, arg2);
1449 }
1450 }
1451
1452 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1453 {
1454 if (TCG_TARGET_HAS_rem_i64) {
1455 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1456 } else if (TCG_TARGET_HAS_div_i64) {
1457 TCGv_i64 t0 = tcg_temp_new_i64();
1458 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1459 tcg_gen_mul_i64(t0, t0, arg2);
1460 tcg_gen_sub_i64(ret, arg1, t0);
1461 tcg_temp_free_i64(t0);
1462 } else if (TCG_TARGET_HAS_div2_i64) {
1463 TCGv_i64 t0 = tcg_temp_new_i64();
1464 tcg_gen_sari_i64(t0, arg1, 63);
1465 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1466 tcg_temp_free_i64(t0);
1467 } else {
1468 gen_helper_rem_i64(ret, arg1, arg2);
1469 }
1470 }
1471
1472 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1473 {
1474 if (TCG_TARGET_HAS_div_i64) {
1475 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1476 } else if (TCG_TARGET_HAS_div2_i64) {
1477 TCGv_i64 t0 = tcg_temp_new_i64();
1478 tcg_gen_movi_i64(t0, 0);
1479 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1480 tcg_temp_free_i64(t0);
1481 } else {
1482 gen_helper_divu_i64(ret, arg1, arg2);
1483 }
1484 }
1485
1486 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1487 {
1488 if (TCG_TARGET_HAS_rem_i64) {
1489 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1490 } else if (TCG_TARGET_HAS_div_i64) {
1491 TCGv_i64 t0 = tcg_temp_new_i64();
1492 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1493 tcg_gen_mul_i64(t0, t0, arg2);
1494 tcg_gen_sub_i64(ret, arg1, t0);
1495 tcg_temp_free_i64(t0);
1496 } else if (TCG_TARGET_HAS_div2_i64) {
1497 TCGv_i64 t0 = tcg_temp_new_i64();
1498 tcg_gen_movi_i64(t0, 0);
1499 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1500 tcg_temp_free_i64(t0);
1501 } else {
1502 gen_helper_remu_i64(ret, arg1, arg2);
1503 }
1504 }
1505
1506 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1507 {
1508 if (TCG_TARGET_REG_BITS == 32) {
1509 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1510 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1511 } else if (TCG_TARGET_HAS_ext8s_i64) {
1512 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1513 } else {
1514 tcg_gen_shli_i64(ret, arg, 56);
1515 tcg_gen_sari_i64(ret, ret, 56);
1516 }
1517 }
1518
1519 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1520 {
1521 if (TCG_TARGET_REG_BITS == 32) {
1522 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1523 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1524 } else if (TCG_TARGET_HAS_ext16s_i64) {
1525 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1526 } else {
1527 tcg_gen_shli_i64(ret, arg, 48);
1528 tcg_gen_sari_i64(ret, ret, 48);
1529 }
1530 }
1531
1532 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1533 {
1534 if (TCG_TARGET_REG_BITS == 32) {
1535 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1536 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1537 } else if (TCG_TARGET_HAS_ext32s_i64) {
1538 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1539 } else {
1540 tcg_gen_shli_i64(ret, arg, 32);
1541 tcg_gen_sari_i64(ret, ret, 32);
1542 }
1543 }
1544
1545 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1546 {
1547 if (TCG_TARGET_REG_BITS == 32) {
1548 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1549 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1550 } else if (TCG_TARGET_HAS_ext8u_i64) {
1551 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1552 } else {
1553 tcg_gen_andi_i64(ret, arg, 0xffu);
1554 }
1555 }
1556
1557 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1558 {
1559 if (TCG_TARGET_REG_BITS == 32) {
1560 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1561 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1562 } else if (TCG_TARGET_HAS_ext16u_i64) {
1563 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1564 } else {
1565 tcg_gen_andi_i64(ret, arg, 0xffffu);
1566 }
1567 }
1568
1569 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1570 {
1571 if (TCG_TARGET_REG_BITS == 32) {
1572 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1573 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1574 } else if (TCG_TARGET_HAS_ext32u_i64) {
1575 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1576 } else {
1577 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1578 }
1579 }
1580
1581 /* Note: we assume the six high bytes are set to zero */
1582 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1583 {
1584 if (TCG_TARGET_REG_BITS == 32) {
1585 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1586 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1587 } else if (TCG_TARGET_HAS_bswap16_i64) {
1588 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1589 } else {
1590 TCGv_i64 t0 = tcg_temp_new_i64();
1591
1592 tcg_gen_ext8u_i64(t0, arg);
1593 tcg_gen_shli_i64(t0, t0, 8);
1594 tcg_gen_shri_i64(ret, arg, 8);
1595 tcg_gen_or_i64(ret, ret, t0);
1596 tcg_temp_free_i64(t0);
1597 }
1598 }
1599
1600 /* Note: we assume the four high bytes are set to zero */
1601 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1602 {
1603 if (TCG_TARGET_REG_BITS == 32) {
1604 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1605 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1606 } else if (TCG_TARGET_HAS_bswap32_i64) {
1607 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1608 } else {
1609 TCGv_i64 t0, t1;
1610 t0 = tcg_temp_new_i64();
1611 t1 = tcg_temp_new_i64();
1612
1613 tcg_gen_shli_i64(t0, arg, 24);
1614 tcg_gen_ext32u_i64(t0, t0);
1615
1616 tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1617 tcg_gen_shli_i64(t1, t1, 8);
1618 tcg_gen_or_i64(t0, t0, t1);
1619
1620 tcg_gen_shri_i64(t1, arg, 8);
1621 tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1622 tcg_gen_or_i64(t0, t0, t1);
1623
1624 tcg_gen_shri_i64(t1, arg, 24);
1625 tcg_gen_or_i64(ret, t0, t1);
1626 tcg_temp_free_i64(t0);
1627 tcg_temp_free_i64(t1);
1628 }
1629 }
1630
1631 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1632 {
1633 if (TCG_TARGET_REG_BITS == 32) {
1634 TCGv_i32 t0, t1;
1635 t0 = tcg_temp_new_i32();
1636 t1 = tcg_temp_new_i32();
1637
1638 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1639 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1640 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1641 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1642 tcg_temp_free_i32(t0);
1643 tcg_temp_free_i32(t1);
1644 } else if (TCG_TARGET_HAS_bswap64_i64) {
1645 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1646 } else {
1647 TCGv_i64 t0 = tcg_temp_new_i64();
1648 TCGv_i64 t1 = tcg_temp_new_i64();
1649
1650 tcg_gen_shli_i64(t0, arg, 56);
1651
1652 tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1653 tcg_gen_shli_i64(t1, t1, 40);
1654 tcg_gen_or_i64(t0, t0, t1);
1655
1656 tcg_gen_andi_i64(t1, arg, 0x00ff0000);
1657 tcg_gen_shli_i64(t1, t1, 24);
1658 tcg_gen_or_i64(t0, t0, t1);
1659
1660 tcg_gen_andi_i64(t1, arg, 0xff000000);
1661 tcg_gen_shli_i64(t1, t1, 8);
1662 tcg_gen_or_i64(t0, t0, t1);
1663
1664 tcg_gen_shri_i64(t1, arg, 8);
1665 tcg_gen_andi_i64(t1, t1, 0xff000000);
1666 tcg_gen_or_i64(t0, t0, t1);
1667
1668 tcg_gen_shri_i64(t1, arg, 24);
1669 tcg_gen_andi_i64(t1, t1, 0x00ff0000);
1670 tcg_gen_or_i64(t0, t0, t1);
1671
1672 tcg_gen_shri_i64(t1, arg, 40);
1673 tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1674 tcg_gen_or_i64(t0, t0, t1);
1675
1676 tcg_gen_shri_i64(t1, arg, 56);
1677 tcg_gen_or_i64(ret, t0, t1);
1678 tcg_temp_free_i64(t0);
1679 tcg_temp_free_i64(t1);
1680 }
1681 }
1682
1683 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1684 {
1685 if (TCG_TARGET_REG_BITS == 32) {
1686 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1687 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1688 } else if (TCG_TARGET_HAS_not_i64) {
1689 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1690 } else {
1691 tcg_gen_xori_i64(ret, arg, -1);
1692 }
1693 }
1694
1695 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1696 {
1697 if (TCG_TARGET_REG_BITS == 32) {
1698 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1699 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1700 } else if (TCG_TARGET_HAS_andc_i64) {
1701 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1702 } else {
1703 TCGv_i64 t0 = tcg_temp_new_i64();
1704 tcg_gen_not_i64(t0, arg2);
1705 tcg_gen_and_i64(ret, arg1, t0);
1706 tcg_temp_free_i64(t0);
1707 }
1708 }
1709
1710 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1711 {
1712 if (TCG_TARGET_REG_BITS == 32) {
1713 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1714 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1715 } else if (TCG_TARGET_HAS_eqv_i64) {
1716 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1717 } else {
1718 tcg_gen_xor_i64(ret, arg1, arg2);
1719 tcg_gen_not_i64(ret, ret);
1720 }
1721 }
1722
1723 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1724 {
1725 if (TCG_TARGET_REG_BITS == 32) {
1726 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1727 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1728 } else if (TCG_TARGET_HAS_nand_i64) {
1729 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1730 } else {
1731 tcg_gen_and_i64(ret, arg1, arg2);
1732 tcg_gen_not_i64(ret, ret);
1733 }
1734 }
1735
1736 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1737 {
1738 if (TCG_TARGET_REG_BITS == 32) {
1739 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1740 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1741 } else if (TCG_TARGET_HAS_nor_i64) {
1742 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1743 } else {
1744 tcg_gen_or_i64(ret, arg1, arg2);
1745 tcg_gen_not_i64(ret, ret);
1746 }
1747 }
1748
1749 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1750 {
1751 if (TCG_TARGET_REG_BITS == 32) {
1752 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1753 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1754 } else if (TCG_TARGET_HAS_orc_i64) {
1755 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1756 } else {
1757 TCGv_i64 t0 = tcg_temp_new_i64();
1758 tcg_gen_not_i64(t0, arg2);
1759 tcg_gen_or_i64(ret, arg1, t0);
1760 tcg_temp_free_i64(t0);
1761 }
1762 }
1763
1764 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1765 {
1766 if (TCG_TARGET_HAS_clz_i64) {
1767 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1768 } else {
1769 gen_helper_clz_i64(ret, arg1, arg2);
1770 }
1771 }
1772
1773 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1774 {
1775 if (TCG_TARGET_REG_BITS == 32
1776 && TCG_TARGET_HAS_clz_i32
1777 && arg2 <= 0xffffffffu) {
1778 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1779 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1780 tcg_gen_addi_i32(t, t, 32);
1781 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1782 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1783 tcg_temp_free_i32(t);
1784 } else {
1785 TCGv_i64 t = tcg_const_i64(arg2);
1786 tcg_gen_clz_i64(ret, arg1, t);
1787 tcg_temp_free_i64(t);
1788 }
1789 }
1790
1791 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1792 {
1793 if (TCG_TARGET_HAS_ctz_i64) {
1794 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1795 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1796 TCGv_i64 z, t = tcg_temp_new_i64();
1797
1798 if (TCG_TARGET_HAS_ctpop_i64) {
1799 tcg_gen_subi_i64(t, arg1, 1);
1800 tcg_gen_andc_i64(t, t, arg1);
1801 tcg_gen_ctpop_i64(t, t);
1802 } else {
1803 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1804 tcg_gen_neg_i64(t, arg1);
1805 tcg_gen_and_i64(t, t, arg1);
1806 tcg_gen_clzi_i64(t, t, 64);
1807 tcg_gen_xori_i64(t, t, 63);
1808 }
1809 z = tcg_const_i64(0);
1810 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1811 tcg_temp_free_i64(t);
1812 tcg_temp_free_i64(z);
1813 } else {
1814 gen_helper_ctz_i64(ret, arg1, arg2);
1815 }
1816 }
1817
1818 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1819 {
1820 if (TCG_TARGET_REG_BITS == 32
1821 && TCG_TARGET_HAS_ctz_i32
1822 && arg2 <= 0xffffffffu) {
1823 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1824 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1825 tcg_gen_addi_i32(t32, t32, 32);
1826 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1827 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1828 tcg_temp_free_i32(t32);
1829 } else if (!TCG_TARGET_HAS_ctz_i64
1830 && TCG_TARGET_HAS_ctpop_i64
1831 && arg2 == 64) {
1832 /* This equivalence has the advantage of not requiring a fixup. */
1833 TCGv_i64 t = tcg_temp_new_i64();
1834 tcg_gen_subi_i64(t, arg1, 1);
1835 tcg_gen_andc_i64(t, t, arg1);
1836 tcg_gen_ctpop_i64(ret, t);
1837 tcg_temp_free_i64(t);
1838 } else {
1839 TCGv_i64 t64 = tcg_const_i64(arg2);
1840 tcg_gen_ctz_i64(ret, arg1, t64);
1841 tcg_temp_free_i64(t64);
1842 }
1843 }
1844
1845 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1846 {
1847 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1848 TCGv_i64 t = tcg_temp_new_i64();
1849 tcg_gen_sari_i64(t, arg, 63);
1850 tcg_gen_xor_i64(t, t, arg);
1851 tcg_gen_clzi_i64(t, t, 64);
1852 tcg_gen_subi_i64(ret, t, 1);
1853 tcg_temp_free_i64(t);
1854 } else {
1855 gen_helper_clrsb_i64(ret, arg);
1856 }
1857 }
1858
1859 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1860 {
1861 if (TCG_TARGET_HAS_ctpop_i64) {
1862 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1863 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1864 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1865 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1866 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1867 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1868 } else {
1869 gen_helper_ctpop_i64(ret, arg1);
1870 }
1871 }
1872
1873 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1874 {
1875 if (TCG_TARGET_HAS_rot_i64) {
1876 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1877 } else {
1878 TCGv_i64 t0, t1;
1879 t0 = tcg_temp_new_i64();
1880 t1 = tcg_temp_new_i64();
1881 tcg_gen_shl_i64(t0, arg1, arg2);
1882 tcg_gen_subfi_i64(t1, 64, arg2);
1883 tcg_gen_shr_i64(t1, arg1, t1);
1884 tcg_gen_or_i64(ret, t0, t1);
1885 tcg_temp_free_i64(t0);
1886 tcg_temp_free_i64(t1);
1887 }
1888 }
1889
1890 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1891 {
1892 tcg_debug_assert(arg2 < 64);
1893 /* some cases can be optimized here */
1894 if (arg2 == 0) {
1895 tcg_gen_mov_i64(ret, arg1);
1896 } else if (TCG_TARGET_HAS_rot_i64) {
1897 TCGv_i64 t0 = tcg_const_i64(arg2);
1898 tcg_gen_rotl_i64(ret, arg1, t0);
1899 tcg_temp_free_i64(t0);
1900 } else {
1901 TCGv_i64 t0, t1;
1902 t0 = tcg_temp_new_i64();
1903 t1 = tcg_temp_new_i64();
1904 tcg_gen_shli_i64(t0, arg1, arg2);
1905 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1906 tcg_gen_or_i64(ret, t0, t1);
1907 tcg_temp_free_i64(t0);
1908 tcg_temp_free_i64(t1);
1909 }
1910 }
1911
1912 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1913 {
1914 if (TCG_TARGET_HAS_rot_i64) {
1915 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1916 } else {
1917 TCGv_i64 t0, t1;
1918 t0 = tcg_temp_new_i64();
1919 t1 = tcg_temp_new_i64();
1920 tcg_gen_shr_i64(t0, arg1, arg2);
1921 tcg_gen_subfi_i64(t1, 64, arg2);
1922 tcg_gen_shl_i64(t1, arg1, t1);
1923 tcg_gen_or_i64(ret, t0, t1);
1924 tcg_temp_free_i64(t0);
1925 tcg_temp_free_i64(t1);
1926 }
1927 }
1928
1929 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1930 {
1931 tcg_debug_assert(arg2 < 64);
1932 /* some cases can be optimized here */
1933 if (arg2 == 0) {
1934 tcg_gen_mov_i64(ret, arg1);
1935 } else {
1936 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1937 }
1938 }
1939
1940 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1941 unsigned int ofs, unsigned int len)
1942 {
1943 uint64_t mask;
1944 TCGv_i64 t1;
1945
1946 tcg_debug_assert(ofs < 64);
1947 tcg_debug_assert(len > 0);
1948 tcg_debug_assert(len <= 64);
1949 tcg_debug_assert(ofs + len <= 64);
1950
1951 if (len == 64) {
1952 tcg_gen_mov_i64(ret, arg2);
1953 return;
1954 }
1955 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1956 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1957 return;
1958 }
1959
1960 if (TCG_TARGET_REG_BITS == 32) {
1961 if (ofs >= 32) {
1962 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1963 TCGV_LOW(arg2), ofs - 32, len);
1964 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1965 return;
1966 }
1967 if (ofs + len <= 32) {
1968 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
1969 TCGV_LOW(arg2), ofs, len);
1970 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1971 return;
1972 }
1973 }
1974
1975 mask = (1ull << len) - 1;
1976 t1 = tcg_temp_new_i64();
1977
1978 if (ofs + len < 64) {
1979 tcg_gen_andi_i64(t1, arg2, mask);
1980 tcg_gen_shli_i64(t1, t1, ofs);
1981 } else {
1982 tcg_gen_shli_i64(t1, arg2, ofs);
1983 }
1984 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
1985 tcg_gen_or_i64(ret, ret, t1);
1986
1987 tcg_temp_free_i64(t1);
1988 }
1989
1990 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
1991 unsigned int ofs, unsigned int len)
1992 {
1993 tcg_debug_assert(ofs < 64);
1994 tcg_debug_assert(len > 0);
1995 tcg_debug_assert(len <= 64);
1996 tcg_debug_assert(ofs + len <= 64);
1997
1998 if (ofs + len == 64) {
1999 tcg_gen_shli_i64(ret, arg, ofs);
2000 } else if (ofs == 0) {
2001 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2002 } else if (TCG_TARGET_HAS_deposit_i64
2003 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2004 TCGv_i64 zero = tcg_const_i64(0);
2005 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2006 tcg_temp_free_i64(zero);
2007 } else {
2008 if (TCG_TARGET_REG_BITS == 32) {
2009 if (ofs >= 32) {
2010 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2011 ofs - 32, len);
2012 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2013 return;
2014 }
2015 if (ofs + len <= 32) {
2016 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2017 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2018 return;
2019 }
2020 }
2021 /* To help two-operand hosts we prefer to zero-extend first,
2022 which allows ARG to stay live. */
2023 switch (len) {
2024 case 32:
2025 if (TCG_TARGET_HAS_ext32u_i64) {
2026 tcg_gen_ext32u_i64(ret, arg);
2027 tcg_gen_shli_i64(ret, ret, ofs);
2028 return;
2029 }
2030 break;
2031 case 16:
2032 if (TCG_TARGET_HAS_ext16u_i64) {
2033 tcg_gen_ext16u_i64(ret, arg);
2034 tcg_gen_shli_i64(ret, ret, ofs);
2035 return;
2036 }
2037 break;
2038 case 8:
2039 if (TCG_TARGET_HAS_ext8u_i64) {
2040 tcg_gen_ext8u_i64(ret, arg);
2041 tcg_gen_shli_i64(ret, ret, ofs);
2042 return;
2043 }
2044 break;
2045 }
2046 /* Otherwise prefer zero-extension over AND for code size. */
2047 switch (ofs + len) {
2048 case 32:
2049 if (TCG_TARGET_HAS_ext32u_i64) {
2050 tcg_gen_shli_i64(ret, arg, ofs);
2051 tcg_gen_ext32u_i64(ret, ret);
2052 return;
2053 }
2054 break;
2055 case 16:
2056 if (TCG_TARGET_HAS_ext16u_i64) {
2057 tcg_gen_shli_i64(ret, arg, ofs);
2058 tcg_gen_ext16u_i64(ret, ret);
2059 return;
2060 }
2061 break;
2062 case 8:
2063 if (TCG_TARGET_HAS_ext8u_i64) {
2064 tcg_gen_shli_i64(ret, arg, ofs);
2065 tcg_gen_ext8u_i64(ret, ret);
2066 return;
2067 }
2068 break;
2069 }
2070 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2071 tcg_gen_shli_i64(ret, ret, ofs);
2072 }
2073 }
2074
2075 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2076 unsigned int ofs, unsigned int len)
2077 {
2078 tcg_debug_assert(ofs < 64);
2079 tcg_debug_assert(len > 0);
2080 tcg_debug_assert(len <= 64);
2081 tcg_debug_assert(ofs + len <= 64);
2082
2083 /* Canonicalize certain special cases, even if extract is supported. */
2084 if (ofs + len == 64) {
2085 tcg_gen_shri_i64(ret, arg, 64 - len);
2086 return;
2087 }
2088 if (ofs == 0) {
2089 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2090 return;
2091 }
2092
2093 if (TCG_TARGET_REG_BITS == 32) {
2094 /* Look for a 32-bit extract within one of the two words. */
2095 if (ofs >= 32) {
2096 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2097 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2098 return;
2099 }
2100 if (ofs + len <= 32) {
2101 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2102 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2103 return;
2104 }
2105 /* The field is split across two words. One double-word
2106 shift is better than two double-word shifts. */
2107 goto do_shift_and;
2108 }
2109
2110 if (TCG_TARGET_HAS_extract_i64
2111 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2112 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2113 return;
2114 }
2115
2116 /* Assume that zero-extension, if available, is cheaper than a shift. */
2117 switch (ofs + len) {
2118 case 32:
2119 if (TCG_TARGET_HAS_ext32u_i64) {
2120 tcg_gen_ext32u_i64(ret, arg);
2121 tcg_gen_shri_i64(ret, ret, ofs);
2122 return;
2123 }
2124 break;
2125 case 16:
2126 if (TCG_TARGET_HAS_ext16u_i64) {
2127 tcg_gen_ext16u_i64(ret, arg);
2128 tcg_gen_shri_i64(ret, ret, ofs);
2129 return;
2130 }
2131 break;
2132 case 8:
2133 if (TCG_TARGET_HAS_ext8u_i64) {
2134 tcg_gen_ext8u_i64(ret, arg);
2135 tcg_gen_shri_i64(ret, ret, ofs);
2136 return;
2137 }
2138 break;
2139 }
2140
2141 /* ??? Ideally we'd know what values are available for immediate AND.
2142 Assume that 8 bits are available, plus the special cases of 16 and 32,
2143 so that we get ext8u, ext16u, and ext32u. */
2144 switch (len) {
2145 case 1 ... 8: case 16: case 32:
2146 do_shift_and:
2147 tcg_gen_shri_i64(ret, arg, ofs);
2148 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2149 break;
2150 default:
2151 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2152 tcg_gen_shri_i64(ret, ret, 64 - len);
2153 break;
2154 }
2155 }
2156
2157 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2158 unsigned int ofs, unsigned int len)
2159 {
2160 tcg_debug_assert(ofs < 64);
2161 tcg_debug_assert(len > 0);
2162 tcg_debug_assert(len <= 64);
2163 tcg_debug_assert(ofs + len <= 64);
2164
2165 /* Canonicalize certain special cases, even if sextract is supported. */
2166 if (ofs + len == 64) {
2167 tcg_gen_sari_i64(ret, arg, 64 - len);
2168 return;
2169 }
2170 if (ofs == 0) {
2171 switch (len) {
2172 case 32:
2173 tcg_gen_ext32s_i64(ret, arg);
2174 return;
2175 case 16:
2176 tcg_gen_ext16s_i64(ret, arg);
2177 return;
2178 case 8:
2179 tcg_gen_ext8s_i64(ret, arg);
2180 return;
2181 }
2182 }
2183
2184 if (TCG_TARGET_REG_BITS == 32) {
2185 /* Look for a 32-bit extract within one of the two words. */
2186 if (ofs >= 32) {
2187 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2188 } else if (ofs + len <= 32) {
2189 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2190 } else if (ofs == 0) {
2191 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2192 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2193 return;
2194 } else if (len > 32) {
2195 TCGv_i32 t = tcg_temp_new_i32();
2196 /* Extract the bits for the high word normally. */
2197 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2198 /* Shift the field down for the low part. */
2199 tcg_gen_shri_i64(ret, arg, ofs);
2200 /* Overwrite the shift into the high part. */
2201 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2202 tcg_temp_free_i32(t);
2203 return;
2204 } else {
2205 /* Shift the field down for the low part, such that the
2206 field sits at the MSB. */
2207 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2208 /* Shift the field down from the MSB, sign extending. */
2209 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2210 }
2211 /* Sign-extend the field from 32 bits. */
2212 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2213 return;
2214 }
2215
2216 if (TCG_TARGET_HAS_sextract_i64
2217 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2218 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2219 return;
2220 }
2221
2222 /* Assume that sign-extension, if available, is cheaper than a shift. */
2223 switch (ofs + len) {
2224 case 32:
2225 if (TCG_TARGET_HAS_ext32s_i64) {
2226 tcg_gen_ext32s_i64(ret, arg);
2227 tcg_gen_sari_i64(ret, ret, ofs);
2228 return;
2229 }
2230 break;
2231 case 16:
2232 if (TCG_TARGET_HAS_ext16s_i64) {
2233 tcg_gen_ext16s_i64(ret, arg);
2234 tcg_gen_sari_i64(ret, ret, ofs);
2235 return;
2236 }
2237 break;
2238 case 8:
2239 if (TCG_TARGET_HAS_ext8s_i64) {
2240 tcg_gen_ext8s_i64(ret, arg);
2241 tcg_gen_sari_i64(ret, ret, ofs);
2242 return;
2243 }
2244 break;
2245 }
2246 switch (len) {
2247 case 32:
2248 if (TCG_TARGET_HAS_ext32s_i64) {
2249 tcg_gen_shri_i64(ret, arg, ofs);
2250 tcg_gen_ext32s_i64(ret, ret);
2251 return;
2252 }
2253 break;
2254 case 16:
2255 if (TCG_TARGET_HAS_ext16s_i64) {
2256 tcg_gen_shri_i64(ret, arg, ofs);
2257 tcg_gen_ext16s_i64(ret, ret);
2258 return;
2259 }
2260 break;
2261 case 8:
2262 if (TCG_TARGET_HAS_ext8s_i64) {
2263 tcg_gen_shri_i64(ret, arg, ofs);
2264 tcg_gen_ext8s_i64(ret, ret);
2265 return;
2266 }
2267 break;
2268 }
2269 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2270 tcg_gen_sari_i64(ret, ret, 64 - len);
2271 }
2272
2273 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2274 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2275 {
2276 if (cond == TCG_COND_ALWAYS) {
2277 tcg_gen_mov_i64(ret, v1);
2278 } else if (cond == TCG_COND_NEVER) {
2279 tcg_gen_mov_i64(ret, v2);
2280 } else if (TCG_TARGET_REG_BITS == 32) {
2281 TCGv_i32 t0 = tcg_temp_new_i32();
2282 TCGv_i32 t1 = tcg_temp_new_i32();
2283 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2284 TCGV_LOW(c1), TCGV_HIGH(c1),
2285 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2286
2287 if (TCG_TARGET_HAS_movcond_i32) {
2288 tcg_gen_movi_i32(t1, 0);
2289 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2290 TCGV_LOW(v1), TCGV_LOW(v2));
2291 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2292 TCGV_HIGH(v1), TCGV_HIGH(v2));
2293 } else {
2294 tcg_gen_neg_i32(t0, t0);
2295
2296 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2297 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2298 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2299
2300 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2301 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2302 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2303 }
2304 tcg_temp_free_i32(t0);
2305 tcg_temp_free_i32(t1);
2306 } else if (TCG_TARGET_HAS_movcond_i64) {
2307 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2308 } else {
2309 TCGv_i64 t0 = tcg_temp_new_i64();
2310 TCGv_i64 t1 = tcg_temp_new_i64();
2311 tcg_gen_setcond_i64(cond, t0, c1, c2);
2312 tcg_gen_neg_i64(t0, t0);
2313 tcg_gen_and_i64(t1, v1, t0);
2314 tcg_gen_andc_i64(ret, v2, t0);
2315 tcg_gen_or_i64(ret, ret, t1);
2316 tcg_temp_free_i64(t0);
2317 tcg_temp_free_i64(t1);
2318 }
2319 }
2320
2321 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2322 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2323 {
2324 if (TCG_TARGET_HAS_add2_i64) {
2325 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2326 } else {
2327 TCGv_i64 t0 = tcg_temp_new_i64();
2328 TCGv_i64 t1 = tcg_temp_new_i64();
2329 tcg_gen_add_i64(t0, al, bl);
2330 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2331 tcg_gen_add_i64(rh, ah, bh);
2332 tcg_gen_add_i64(rh, rh, t1);
2333 tcg_gen_mov_i64(rl, t0);
2334 tcg_temp_free_i64(t0);
2335 tcg_temp_free_i64(t1);
2336 }
2337 }
2338
2339 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2340 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2341 {
2342 if (TCG_TARGET_HAS_sub2_i64) {
2343 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2344 } else {
2345 TCGv_i64 t0 = tcg_temp_new_i64();
2346 TCGv_i64 t1 = tcg_temp_new_i64();
2347 tcg_gen_sub_i64(t0, al, bl);
2348 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2349 tcg_gen_sub_i64(rh, ah, bh);
2350 tcg_gen_sub_i64(rh, rh, t1);
2351 tcg_gen_mov_i64(rl, t0);
2352 tcg_temp_free_i64(t0);
2353 tcg_temp_free_i64(t1);
2354 }
2355 }
2356
2357 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2358 {
2359 if (TCG_TARGET_HAS_mulu2_i64) {
2360 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2361 } else if (TCG_TARGET_HAS_muluh_i64) {
2362 TCGv_i64 t = tcg_temp_new_i64();
2363 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2364 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2365 tcg_gen_mov_i64(rl, t);
2366 tcg_temp_free_i64(t);
2367 } else {
2368 TCGv_i64 t0 = tcg_temp_new_i64();
2369 tcg_gen_mul_i64(t0, arg1, arg2);
2370 gen_helper_muluh_i64(rh, arg1, arg2);
2371 tcg_gen_mov_i64(rl, t0);
2372 tcg_temp_free_i64(t0);
2373 }
2374 }
2375
2376 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2377 {
2378 if (TCG_TARGET_HAS_muls2_i64) {
2379 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2380 } else if (TCG_TARGET_HAS_mulsh_i64) {
2381 TCGv_i64 t = tcg_temp_new_i64();
2382 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2383 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2384 tcg_gen_mov_i64(rl, t);
2385 tcg_temp_free_i64(t);
2386 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2387 TCGv_i64 t0 = tcg_temp_new_i64();
2388 TCGv_i64 t1 = tcg_temp_new_i64();
2389 TCGv_i64 t2 = tcg_temp_new_i64();
2390 TCGv_i64 t3 = tcg_temp_new_i64();
2391 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2392 /* Adjust for negative inputs. */
2393 tcg_gen_sari_i64(t2, arg1, 63);
2394 tcg_gen_sari_i64(t3, arg2, 63);
2395 tcg_gen_and_i64(t2, t2, arg2);
2396 tcg_gen_and_i64(t3, t3, arg1);
2397 tcg_gen_sub_i64(rh, t1, t2);
2398 tcg_gen_sub_i64(rh, rh, t3);
2399 tcg_gen_mov_i64(rl, t0);
2400 tcg_temp_free_i64(t0);
2401 tcg_temp_free_i64(t1);
2402 tcg_temp_free_i64(t2);
2403 tcg_temp_free_i64(t3);
2404 } else {
2405 TCGv_i64 t0 = tcg_temp_new_i64();
2406 tcg_gen_mul_i64(t0, arg1, arg2);
2407 gen_helper_mulsh_i64(rh, arg1, arg2);
2408 tcg_gen_mov_i64(rl, t0);
2409 tcg_temp_free_i64(t0);
2410 }
2411 }
2412
2413 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2414 {
2415 TCGv_i64 t0 = tcg_temp_new_i64();
2416 TCGv_i64 t1 = tcg_temp_new_i64();
2417 TCGv_i64 t2 = tcg_temp_new_i64();
2418 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2419 /* Adjust for negative input for the signed arg1. */
2420 tcg_gen_sari_i64(t2, arg1, 63);
2421 tcg_gen_and_i64(t2, t2, arg2);
2422 tcg_gen_sub_i64(rh, t1, t2);
2423 tcg_gen_mov_i64(rl, t0);
2424 tcg_temp_free_i64(t0);
2425 tcg_temp_free_i64(t1);
2426 tcg_temp_free_i64(t2);
2427 }
2428
2429 /* Size changing operations. */
2430
2431 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2432 {
2433 if (TCG_TARGET_REG_BITS == 32) {
2434 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2435 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2436 tcg_gen_op2(INDEX_op_extrl_i64_i32,
2437 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2438 } else {
2439 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2440 }
2441 }
2442
2443 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2444 {
2445 if (TCG_TARGET_REG_BITS == 32) {
2446 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2447 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2448 tcg_gen_op2(INDEX_op_extrh_i64_i32,
2449 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2450 } else {
2451 TCGv_i64 t = tcg_temp_new_i64();
2452 tcg_gen_shri_i64(t, arg, 32);
2453 tcg_gen_mov_i32(ret, (TCGv_i32)t);
2454 tcg_temp_free_i64(t);
2455 }
2456 }
2457
2458 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2459 {
2460 if (TCG_TARGET_REG_BITS == 32) {
2461 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2462 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2463 } else {
2464 tcg_gen_op2(INDEX_op_extu_i32_i64,
2465 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2466 }
2467 }
2468
2469 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2470 {
2471 if (TCG_TARGET_REG_BITS == 32) {
2472 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2473 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2474 } else {
2475 tcg_gen_op2(INDEX_op_ext_i32_i64,
2476 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2477 }
2478 }
2479
2480 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2481 {
2482 TCGv_i64 tmp;
2483
2484 if (TCG_TARGET_REG_BITS == 32) {
2485 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2486 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2487 return;
2488 }
2489
2490 tmp = tcg_temp_new_i64();
2491 /* These extensions are only needed for type correctness.
2492 We may be able to do better given target specific information. */
2493 tcg_gen_extu_i32_i64(tmp, high);
2494 tcg_gen_extu_i32_i64(dest, low);
2495 /* If deposit is available, use it. Otherwise use the extra
2496 knowledge that we have of the zero-extensions above. */
2497 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2498 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2499 } else {
2500 tcg_gen_shli_i64(tmp, tmp, 32);
2501 tcg_gen_or_i64(dest, dest, tmp);
2502 }
2503 tcg_temp_free_i64(tmp);
2504 }
2505
2506 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2507 {
2508 if (TCG_TARGET_REG_BITS == 32) {
2509 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2510 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2511 } else {
2512 tcg_gen_extrl_i64_i32(lo, arg);
2513 tcg_gen_extrh_i64_i32(hi, arg);
2514 }
2515 }
2516
2517 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2518 {
2519 tcg_gen_ext32u_i64(lo, arg);
2520 tcg_gen_shri_i64(hi, arg, 32);
2521 }
2522
2523 /* QEMU specific operations. */
2524
2525 void tcg_gen_goto_tb(unsigned idx)
2526 {
2527 /* We only support two chained exits. */
2528 tcg_debug_assert(idx <= 1);
2529 #ifdef CONFIG_DEBUG_TCG
2530 /* Verify that we havn't seen this numbered exit before. */
2531 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2532 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2533 #endif
2534 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2535 }
2536
2537 void tcg_gen_lookup_and_goto_ptr(void)
2538 {
2539 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2540 TCGv_ptr ptr = tcg_temp_new_ptr();
2541 gen_helper_lookup_tb_ptr(ptr, cpu_env);
2542 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2543 tcg_temp_free_ptr(ptr);
2544 } else {
2545 tcg_gen_exit_tb(0);
2546 }
2547 }
2548
2549 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2550 {
2551 /* Trigger the asserts within as early as possible. */
2552 (void)get_alignment_bits(op);
2553
2554 switch (op & MO_SIZE) {
2555 case MO_8:
2556 op &= ~MO_BSWAP;
2557 break;
2558 case MO_16:
2559 break;
2560 case MO_32:
2561 if (!is64) {
2562 op &= ~MO_SIGN;
2563 }
2564 break;
2565 case MO_64:
2566 if (!is64) {
2567 tcg_abort();
2568 }
2569 break;
2570 }
2571 if (st) {
2572 op &= ~MO_SIGN;
2573 }
2574 return op;
2575 }
2576
2577 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2578 TCGMemOp memop, TCGArg idx)
2579 {
2580 TCGMemOpIdx oi = make_memop_idx(memop, idx);
2581 #if TARGET_LONG_BITS == 32
2582 tcg_gen_op3i_i32(opc, val, addr, oi);
2583 #else
2584 if (TCG_TARGET_REG_BITS == 32) {
2585 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2586 } else {
2587 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2588 }
2589 #endif
2590 }
2591
2592 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2593 TCGMemOp memop, TCGArg idx)
2594 {
2595 TCGMemOpIdx oi = make_memop_idx(memop, idx);
2596 #if TARGET_LONG_BITS == 32
2597 if (TCG_TARGET_REG_BITS == 32) {
2598 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2599 } else {
2600 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2601 }
2602 #else
2603 if (TCG_TARGET_REG_BITS == 32) {
2604 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2605 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2606 } else {
2607 tcg_gen_op3i_i64(opc, val, addr, oi);
2608 }
2609 #endif
2610 }
2611
2612 static void tcg_gen_req_mo(TCGBar type)
2613 {
2614 #ifdef TCG_GUEST_DEFAULT_MO
2615 type &= TCG_GUEST_DEFAULT_MO;
2616 #endif
2617 type &= ~TCG_TARGET_DEFAULT_MO;
2618 if (type) {
2619 tcg_gen_mb(type | TCG_BAR_SC);
2620 }
2621 }
2622
2623 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2624 {
2625 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2626 memop = tcg_canonicalize_memop(memop, 0, 0);
2627 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2628 addr, trace_mem_get_info(memop, 0));
2629 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2630 }
2631
2632 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2633 {
2634 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2635 memop = tcg_canonicalize_memop(memop, 0, 1);
2636 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2637 addr, trace_mem_get_info(memop, 1));
2638 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2639 }
2640
2641 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2642 {
2643 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2644 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2645 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2646 if (memop & MO_SIGN) {
2647 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2648 } else {
2649 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2650 }
2651 return;
2652 }
2653
2654 memop = tcg_canonicalize_memop(memop, 1, 0);
2655 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2656 addr, trace_mem_get_info(memop, 0));
2657 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2658 }
2659
2660 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2661 {
2662 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2663 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2664 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2665 return;
2666 }
2667
2668 memop = tcg_canonicalize_memop(memop, 1, 1);
2669 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2670 addr, trace_mem_get_info(memop, 1));
2671 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
2672 }
2673
2674 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2675 {
2676 switch (opc & MO_SSIZE) {
2677 case MO_SB:
2678 tcg_gen_ext8s_i32(ret, val);
2679 break;
2680 case MO_UB:
2681 tcg_gen_ext8u_i32(ret, val);
2682 break;
2683 case MO_SW:
2684 tcg_gen_ext16s_i32(ret, val);
2685 break;
2686 case MO_UW:
2687 tcg_gen_ext16u_i32(ret, val);
2688 break;
2689 default:
2690 tcg_gen_mov_i32(ret, val);
2691 break;
2692 }
2693 }
2694
2695 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2696 {
2697 switch (opc & MO_SSIZE) {
2698 case MO_SB:
2699 tcg_gen_ext8s_i64(ret, val);
2700 break;
2701 case MO_UB:
2702 tcg_gen_ext8u_i64(ret, val);
2703 break;
2704 case MO_SW:
2705 tcg_gen_ext16s_i64(ret, val);
2706 break;
2707 case MO_UW:
2708 tcg_gen_ext16u_i64(ret, val);
2709 break;
2710 case MO_SL:
2711 tcg_gen_ext32s_i64(ret, val);
2712 break;
2713 case MO_UL:
2714 tcg_gen_ext32u_i64(ret, val);
2715 break;
2716 default:
2717 tcg_gen_mov_i64(ret, val);
2718 break;
2719 }
2720 }
2721
2722 #ifdef CONFIG_SOFTMMU
2723 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2724 TCGv_i32, TCGv_i32, TCGv_i32);
2725 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2726 TCGv_i64, TCGv_i64, TCGv_i32);
2727 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2728 TCGv_i32, TCGv_i32);
2729 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2730 TCGv_i64, TCGv_i32);
2731 #else
2732 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2733 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2734 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2735 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2736 #endif
2737
2738 #ifdef CONFIG_ATOMIC64
2739 # define WITH_ATOMIC64(X) X,
2740 #else
2741 # define WITH_ATOMIC64(X)
2742 #endif
2743
2744 static void * const table_cmpxchg[16] = {
2745 [MO_8] = gen_helper_atomic_cmpxchgb,
2746 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2747 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2748 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2749 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
2750 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2751 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
2752 };
2753
2754 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2755 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2756 {
2757 memop = tcg_canonicalize_memop(memop, 0, 0);
2758
2759 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
2760 TCGv_i32 t1 = tcg_temp_new_i32();
2761 TCGv_i32 t2 = tcg_temp_new_i32();
2762
2763 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2764
2765 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2766 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2767 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2768 tcg_temp_free_i32(t2);
2769
2770 if (memop & MO_SIGN) {
2771 tcg_gen_ext_i32(retv, t1, memop);
2772 } else {
2773 tcg_gen_mov_i32(retv, t1);
2774 }
2775 tcg_temp_free_i32(t1);
2776 } else {
2777 gen_atomic_cx_i32 gen;
2778
2779 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2780 tcg_debug_assert(gen != NULL);
2781
2782 #ifdef CONFIG_SOFTMMU
2783 {
2784 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2785 gen(retv, cpu_env, addr, cmpv, newv, oi);
2786 tcg_temp_free_i32(oi);
2787 }
2788 #else
2789 gen(retv, cpu_env, addr, cmpv, newv);
2790 #endif
2791
2792 if (memop & MO_SIGN) {
2793 tcg_gen_ext_i32(retv, retv, memop);
2794 }
2795 }
2796 }
2797
2798 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2799 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2800 {
2801 memop = tcg_canonicalize_memop(memop, 1, 0);
2802
2803 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
2804 TCGv_i64 t1 = tcg_temp_new_i64();
2805 TCGv_i64 t2 = tcg_temp_new_i64();
2806
2807 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2808
2809 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2810 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2811 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2812 tcg_temp_free_i64(t2);
2813
2814 if (memop & MO_SIGN) {
2815 tcg_gen_ext_i64(retv, t1, memop);
2816 } else {
2817 tcg_gen_mov_i64(retv, t1);
2818 }
2819 tcg_temp_free_i64(t1);
2820 } else if ((memop & MO_SIZE) == MO_64) {
2821 #ifdef CONFIG_ATOMIC64
2822 gen_atomic_cx_i64 gen;
2823
2824 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2825 tcg_debug_assert(gen != NULL);
2826
2827 #ifdef CONFIG_SOFTMMU
2828 {
2829 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
2830 gen(retv, cpu_env, addr, cmpv, newv, oi);
2831 tcg_temp_free_i32(oi);
2832 }
2833 #else
2834 gen(retv, cpu_env, addr, cmpv, newv);
2835 #endif
2836 #else
2837 gen_helper_exit_atomic(cpu_env);
2838 /* Produce a result, so that we have a well-formed opcode stream
2839 with respect to uses of the result in the (dead) code following. */
2840 tcg_gen_movi_i64(retv, 0);
2841 #endif /* CONFIG_ATOMIC64 */
2842 } else {
2843 TCGv_i32 c32 = tcg_temp_new_i32();
2844 TCGv_i32 n32 = tcg_temp_new_i32();
2845 TCGv_i32 r32 = tcg_temp_new_i32();
2846
2847 tcg_gen_extrl_i64_i32(c32, cmpv);
2848 tcg_gen_extrl_i64_i32(n32, newv);
2849 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
2850 tcg_temp_free_i32(c32);
2851 tcg_temp_free_i32(n32);
2852
2853 tcg_gen_extu_i32_i64(retv, r32);
2854 tcg_temp_free_i32(r32);
2855
2856 if (memop & MO_SIGN) {
2857 tcg_gen_ext_i64(retv, retv, memop);
2858 }
2859 }
2860 }
2861
2862 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2863 TCGArg idx, TCGMemOp memop, bool new_val,
2864 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
2865 {
2866 TCGv_i32 t1 = tcg_temp_new_i32();
2867 TCGv_i32 t2 = tcg_temp_new_i32();
2868
2869 memop = tcg_canonicalize_memop(memop, 0, 0);
2870
2871 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2872 gen(t2, t1, val);
2873 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2874
2875 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
2876 tcg_temp_free_i32(t1);
2877 tcg_temp_free_i32(t2);
2878 }
2879
2880 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2881 TCGArg idx, TCGMemOp memop, void * const table[])
2882 {
2883 gen_atomic_op_i32 gen;
2884
2885 memop = tcg_canonicalize_memop(memop, 0, 0);
2886
2887 gen = table[memop & (MO_SIZE | MO_BSWAP)];
2888 tcg_debug_assert(gen != NULL);
2889
2890 #ifdef CONFIG_SOFTMMU
2891 {
2892 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2893 gen(ret, cpu_env, addr, val, oi);
2894 tcg_temp_free_i32(oi);
2895 }
2896 #else
2897 gen(ret, cpu_env, addr, val);
2898 #endif
2899
2900 if (memop & MO_SIGN) {
2901 tcg_gen_ext_i32(ret, ret, memop);
2902 }
2903 }
2904
2905 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2906 TCGArg idx, TCGMemOp memop, bool new_val,
2907 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
2908 {
2909 TCGv_i64 t1 = tcg_temp_new_i64();
2910 TCGv_i64 t2 = tcg_temp_new_i64();
2911
2912 memop = tcg_canonicalize_memop(memop, 1, 0);
2913
2914 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2915 gen(t2, t1, val);
2916 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2917
2918 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
2919 tcg_temp_free_i64(t1);
2920 tcg_temp_free_i64(t2);
2921 }
2922
2923 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2924 TCGArg idx, TCGMemOp memop, void * const table[])
2925 {
2926 memop = tcg_canonicalize_memop(memop, 1, 0);
2927
2928 if ((memop & MO_SIZE) == MO_64) {
2929 #ifdef CONFIG_ATOMIC64
2930 gen_atomic_op_i64 gen;
2931
2932 gen = table[memop & (MO_SIZE | MO_BSWAP)];
2933 tcg_debug_assert(gen != NULL);
2934
2935 #ifdef CONFIG_SOFTMMU
2936 {
2937 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2938 gen(ret, cpu_env, addr, val, oi);
2939 tcg_temp_free_i32(oi);
2940 }
2941 #else
2942 gen(ret, cpu_env, addr, val);
2943 #endif
2944 #else
2945 gen_helper_exit_atomic(cpu_env);
2946 /* Produce a result, so that we have a well-formed opcode stream
2947 with respect to uses of the result in the (dead) code following. */
2948 tcg_gen_movi_i64(ret, 0);
2949 #endif /* CONFIG_ATOMIC64 */
2950 } else {
2951 TCGv_i32 v32 = tcg_temp_new_i32();
2952 TCGv_i32 r32 = tcg_temp_new_i32();
2953
2954 tcg_gen_extrl_i64_i32(v32, val);
2955 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
2956 tcg_temp_free_i32(v32);
2957
2958 tcg_gen_extu_i32_i64(ret, r32);
2959 tcg_temp_free_i32(r32);
2960
2961 if (memop & MO_SIGN) {
2962 tcg_gen_ext_i64(ret, ret, memop);
2963 }
2964 }
2965 }
2966
2967 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
2968 static void * const table_##NAME[16] = { \
2969 [MO_8] = gen_helper_atomic_##NAME##b, \
2970 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
2971 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
2972 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
2973 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
2974 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
2975 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
2976 }; \
2977 void tcg_gen_atomic_##NAME##_i32 \
2978 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
2979 { \
2980 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
2981 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
2982 } else { \
2983 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
2984 tcg_gen_##OP##_i32); \
2985 } \
2986 } \
2987 void tcg_gen_atomic_##NAME##_i64 \
2988 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
2989 { \
2990 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
2991 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
2992 } else { \
2993 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
2994 tcg_gen_##OP##_i64); \
2995 } \
2996 }
2997
2998 GEN_ATOMIC_HELPER(fetch_add, add, 0)
2999 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3000 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3001 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3002
3003 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3004 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3005 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3006 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3007
3008 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3009 {
3010 tcg_gen_mov_i32(r, b);
3011 }
3012
3013 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3014 {
3015 tcg_gen_mov_i64(r, b);
3016 }
3017
3018 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3019
3020 #undef GEN_ATOMIC_HELPER
QEmu