?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
xen: introduce 'xen-block', 'xen-disk' and 'xen-cdrom'
[qemu.git] / tcg / tcg-op.c
blob1bd7ef24affc91b11206a7e7cb1402082b451231
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "qemu-common.h"
27 #include "cpu.h"
28 #include "exec/exec-all.h"
29 #include "tcg.h"
30 #include "tcg-op.h"
31 #include "tcg-mo.h"
32 #include "trace-tcg.h"
33 #include "trace/mem.h"
34
35 /* Reduce the number of ifdefs below. This assumes that all uses of
36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
37 the compiler can eliminate. */
38 #if TCG_TARGET_REG_BITS == 64
39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
41 #define TCGV_LOW TCGV_LOW_link_error
42 #define TCGV_HIGH TCGV_HIGH_link_error
43 #endif
44
45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
46 {
47 TCGOp *op = tcg_emit_op(opc);
48 op->args[0] = a1;
49 }
50
51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
52 {
53 TCGOp *op = tcg_emit_op(opc);
54 op->args[0] = a1;
55 op->args[1] = a2;
56 }
57
58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
59 {
60 TCGOp *op = tcg_emit_op(opc);
61 op->args[0] = a1;
62 op->args[1] = a2;
63 op->args[2] = a3;
64 }
65
66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
67 {
68 TCGOp *op = tcg_emit_op(opc);
69 op->args[0] = a1;
70 op->args[1] = a2;
71 op->args[2] = a3;
72 op->args[3] = a4;
73 }
74
75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76 TCGArg a4, TCGArg a5)
77 {
78 TCGOp *op = tcg_emit_op(opc);
79 op->args[0] = a1;
80 op->args[1] = a2;
81 op->args[2] = a3;
82 op->args[3] = a4;
83 op->args[4] = a5;
84 }
85
86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
87 TCGArg a4, TCGArg a5, TCGArg a6)
88 {
89 TCGOp *op = tcg_emit_op(opc);
90 op->args[0] = a1;
91 op->args[1] = a2;
92 op->args[2] = a3;
93 op->args[3] = a4;
94 op->args[4] = a5;
95 op->args[5] = a6;
96 }
97
98 void tcg_gen_mb(TCGBar mb_type)
99 {
100 if (tcg_ctx->tb_cflags & CF_PARALLEL) {
101 tcg_gen_op1(INDEX_op_mb, mb_type);
102 }
103 }
104
105 /* 32 bit ops */
106
107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
108 {
109 /* some cases can be optimized here */
110 if (arg2 == 0) {
111 tcg_gen_mov_i32(ret, arg1);
112 } else {
113 TCGv_i32 t0 = tcg_const_i32(arg2);
114 tcg_gen_add_i32(ret, arg1, t0);
115 tcg_temp_free_i32(t0);
116 }
117 }
118
119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120 {
121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
122 /* Don't recurse with tcg_gen_neg_i32. */
123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
124 } else {
125 TCGv_i32 t0 = tcg_const_i32(arg1);
126 tcg_gen_sub_i32(ret, t0, arg2);
127 tcg_temp_free_i32(t0);
128 }
129 }
130
131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
132 {
133 /* some cases can be optimized here */
134 if (arg2 == 0) {
135 tcg_gen_mov_i32(ret, arg1);
136 } else {
137 TCGv_i32 t0 = tcg_const_i32(arg2);
138 tcg_gen_sub_i32(ret, arg1, t0);
139 tcg_temp_free_i32(t0);
140 }
141 }
142
143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
144 {
145 TCGv_i32 t0;
146 /* Some cases can be optimized here. */
147 switch (arg2) {
148 case 0:
149 tcg_gen_movi_i32(ret, 0);
150 return;
151 case -1:
152 tcg_gen_mov_i32(ret, arg1);
153 return;
154 case 0xff:
155 /* Don't recurse with tcg_gen_ext8u_i32. */
156 if (TCG_TARGET_HAS_ext8u_i32) {
157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
158 return;
159 }
160 break;
161 case 0xffff:
162 if (TCG_TARGET_HAS_ext16u_i32) {
163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
164 return;
165 }
166 break;
167 }
168 t0 = tcg_const_i32(arg2);
169 tcg_gen_and_i32(ret, arg1, t0);
170 tcg_temp_free_i32(t0);
171 }
172
173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
174 {
175 /* Some cases can be optimized here. */
176 if (arg2 == -1) {
177 tcg_gen_movi_i32(ret, -1);
178 } else if (arg2 == 0) {
179 tcg_gen_mov_i32(ret, arg1);
180 } else {
181 TCGv_i32 t0 = tcg_const_i32(arg2);
182 tcg_gen_or_i32(ret, arg1, t0);
183 tcg_temp_free_i32(t0);
184 }
185 }
186
187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
188 {
189 /* Some cases can be optimized here. */
190 if (arg2 == 0) {
191 tcg_gen_mov_i32(ret, arg1);
192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
193 /* Don't recurse with tcg_gen_not_i32. */
194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
195 } else {
196 TCGv_i32 t0 = tcg_const_i32(arg2);
197 tcg_gen_xor_i32(ret, arg1, t0);
198 tcg_temp_free_i32(t0);
199 }
200 }
201
202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
203 {
204 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
205 if (arg2 == 0) {
206 tcg_gen_mov_i32(ret, arg1);
207 } else {
208 TCGv_i32 t0 = tcg_const_i32(arg2);
209 tcg_gen_shl_i32(ret, arg1, t0);
210 tcg_temp_free_i32(t0);
211 }
212 }
213
214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
215 {
216 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
217 if (arg2 == 0) {
218 tcg_gen_mov_i32(ret, arg1);
219 } else {
220 TCGv_i32 t0 = tcg_const_i32(arg2);
221 tcg_gen_shr_i32(ret, arg1, t0);
222 tcg_temp_free_i32(t0);
223 }
224 }
225
226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
227 {
228 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
229 if (arg2 == 0) {
230 tcg_gen_mov_i32(ret, arg1);
231 } else {
232 TCGv_i32 t0 = tcg_const_i32(arg2);
233 tcg_gen_sar_i32(ret, arg1, t0);
234 tcg_temp_free_i32(t0);
235 }
236 }
237
238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
239 {
240 if (cond == TCG_COND_ALWAYS) {
241 tcg_gen_br(l);
242 } else if (cond != TCG_COND_NEVER) {
243 l->refs++;
244 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
245 }
246 }
247
248 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
249 {
250 if (cond == TCG_COND_ALWAYS) {
251 tcg_gen_br(l);
252 } else if (cond != TCG_COND_NEVER) {
253 TCGv_i32 t0 = tcg_const_i32(arg2);
254 tcg_gen_brcond_i32(cond, arg1, t0, l);
255 tcg_temp_free_i32(t0);
256 }
257 }
258
259 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
260 TCGv_i32 arg1, TCGv_i32 arg2)
261 {
262 if (cond == TCG_COND_ALWAYS) {
263 tcg_gen_movi_i32(ret, 1);
264 } else if (cond == TCG_COND_NEVER) {
265 tcg_gen_movi_i32(ret, 0);
266 } else {
267 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
268 }
269 }
270
271 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
272 TCGv_i32 arg1, int32_t arg2)
273 {
274 TCGv_i32 t0 = tcg_const_i32(arg2);
275 tcg_gen_setcond_i32(cond, ret, arg1, t0);
276 tcg_temp_free_i32(t0);
277 }
278
279 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
280 {
281 if (arg2 == 0) {
282 tcg_gen_movi_i32(ret, 0);
283 } else if (is_power_of_2(arg2)) {
284 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
285 } else {
286 TCGv_i32 t0 = tcg_const_i32(arg2);
287 tcg_gen_mul_i32(ret, arg1, t0);
288 tcg_temp_free_i32(t0);
289 }
290 }
291
292 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
293 {
294 if (TCG_TARGET_HAS_div_i32) {
295 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
296 } else if (TCG_TARGET_HAS_div2_i32) {
297 TCGv_i32 t0 = tcg_temp_new_i32();
298 tcg_gen_sari_i32(t0, arg1, 31);
299 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
300 tcg_temp_free_i32(t0);
301 } else {
302 gen_helper_div_i32(ret, arg1, arg2);
303 }
304 }
305
306 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
307 {
308 if (TCG_TARGET_HAS_rem_i32) {
309 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
310 } else if (TCG_TARGET_HAS_div_i32) {
311 TCGv_i32 t0 = tcg_temp_new_i32();
312 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
313 tcg_gen_mul_i32(t0, t0, arg2);
314 tcg_gen_sub_i32(ret, arg1, t0);
315 tcg_temp_free_i32(t0);
316 } else if (TCG_TARGET_HAS_div2_i32) {
317 TCGv_i32 t0 = tcg_temp_new_i32();
318 tcg_gen_sari_i32(t0, arg1, 31);
319 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
320 tcg_temp_free_i32(t0);
321 } else {
322 gen_helper_rem_i32(ret, arg1, arg2);
323 }
324 }
325
326 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
327 {
328 if (TCG_TARGET_HAS_div_i32) {
329 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
330 } else if (TCG_TARGET_HAS_div2_i32) {
331 TCGv_i32 t0 = tcg_temp_new_i32();
332 tcg_gen_movi_i32(t0, 0);
333 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
334 tcg_temp_free_i32(t0);
335 } else {
336 gen_helper_divu_i32(ret, arg1, arg2);
337 }
338 }
339
340 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
341 {
342 if (TCG_TARGET_HAS_rem_i32) {
343 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
344 } else if (TCG_TARGET_HAS_div_i32) {
345 TCGv_i32 t0 = tcg_temp_new_i32();
346 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
347 tcg_gen_mul_i32(t0, t0, arg2);
348 tcg_gen_sub_i32(ret, arg1, t0);
349 tcg_temp_free_i32(t0);
350 } else if (TCG_TARGET_HAS_div2_i32) {
351 TCGv_i32 t0 = tcg_temp_new_i32();
352 tcg_gen_movi_i32(t0, 0);
353 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
354 tcg_temp_free_i32(t0);
355 } else {
356 gen_helper_remu_i32(ret, arg1, arg2);
357 }
358 }
359
360 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
361 {
362 if (TCG_TARGET_HAS_andc_i32) {
363 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
364 } else {
365 TCGv_i32 t0 = tcg_temp_new_i32();
366 tcg_gen_not_i32(t0, arg2);
367 tcg_gen_and_i32(ret, arg1, t0);
368 tcg_temp_free_i32(t0);
369 }
370 }
371
372 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
373 {
374 if (TCG_TARGET_HAS_eqv_i32) {
375 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
376 } else {
377 tcg_gen_xor_i32(ret, arg1, arg2);
378 tcg_gen_not_i32(ret, ret);
379 }
380 }
381
382 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
383 {
384 if (TCG_TARGET_HAS_nand_i32) {
385 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
386 } else {
387 tcg_gen_and_i32(ret, arg1, arg2);
388 tcg_gen_not_i32(ret, ret);
389 }
390 }
391
392 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
393 {
394 if (TCG_TARGET_HAS_nor_i32) {
395 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
396 } else {
397 tcg_gen_or_i32(ret, arg1, arg2);
398 tcg_gen_not_i32(ret, ret);
399 }
400 }
401
402 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
403 {
404 if (TCG_TARGET_HAS_orc_i32) {
405 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
406 } else {
407 TCGv_i32 t0 = tcg_temp_new_i32();
408 tcg_gen_not_i32(t0, arg2);
409 tcg_gen_or_i32(ret, arg1, t0);
410 tcg_temp_free_i32(t0);
411 }
412 }
413
414 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
415 {
416 if (TCG_TARGET_HAS_clz_i32) {
417 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
418 } else if (TCG_TARGET_HAS_clz_i64) {
419 TCGv_i64 t1 = tcg_temp_new_i64();
420 TCGv_i64 t2 = tcg_temp_new_i64();
421 tcg_gen_extu_i32_i64(t1, arg1);
422 tcg_gen_extu_i32_i64(t2, arg2);
423 tcg_gen_addi_i64(t2, t2, 32);
424 tcg_gen_clz_i64(t1, t1, t2);
425 tcg_gen_extrl_i64_i32(ret, t1);
426 tcg_temp_free_i64(t1);
427 tcg_temp_free_i64(t2);
428 tcg_gen_subi_i32(ret, ret, 32);
429 } else {
430 gen_helper_clz_i32(ret, arg1, arg2);
431 }
432 }
433
434 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
435 {
436 TCGv_i32 t = tcg_const_i32(arg2);
437 tcg_gen_clz_i32(ret, arg1, t);
438 tcg_temp_free_i32(t);
439 }
440
441 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
442 {
443 if (TCG_TARGET_HAS_ctz_i32) {
444 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
445 } else if (TCG_TARGET_HAS_ctz_i64) {
446 TCGv_i64 t1 = tcg_temp_new_i64();
447 TCGv_i64 t2 = tcg_temp_new_i64();
448 tcg_gen_extu_i32_i64(t1, arg1);
449 tcg_gen_extu_i32_i64(t2, arg2);
450 tcg_gen_ctz_i64(t1, t1, t2);
451 tcg_gen_extrl_i64_i32(ret, t1);
452 tcg_temp_free_i64(t1);
453 tcg_temp_free_i64(t2);
454 } else if (TCG_TARGET_HAS_ctpop_i32
455 || TCG_TARGET_HAS_ctpop_i64
456 || TCG_TARGET_HAS_clz_i32
457 || TCG_TARGET_HAS_clz_i64) {
458 TCGv_i32 z, t = tcg_temp_new_i32();
459
460 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
461 tcg_gen_subi_i32(t, arg1, 1);
462 tcg_gen_andc_i32(t, t, arg1);
463 tcg_gen_ctpop_i32(t, t);
464 } else {
465 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
466 tcg_gen_neg_i32(t, arg1);
467 tcg_gen_and_i32(t, t, arg1);
468 tcg_gen_clzi_i32(t, t, 32);
469 tcg_gen_xori_i32(t, t, 31);
470 }
471 z = tcg_const_i32(0);
472 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
473 tcg_temp_free_i32(t);
474 tcg_temp_free_i32(z);
475 } else {
476 gen_helper_ctz_i32(ret, arg1, arg2);
477 }
478 }
479
480 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
481 {
482 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
483 /* This equivalence has the advantage of not requiring a fixup. */
484 TCGv_i32 t = tcg_temp_new_i32();
485 tcg_gen_subi_i32(t, arg1, 1);
486 tcg_gen_andc_i32(t, t, arg1);
487 tcg_gen_ctpop_i32(ret, t);
488 tcg_temp_free_i32(t);
489 } else {
490 TCGv_i32 t = tcg_const_i32(arg2);
491 tcg_gen_ctz_i32(ret, arg1, t);
492 tcg_temp_free_i32(t);
493 }
494 }
495
496 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
497 {
498 if (TCG_TARGET_HAS_clz_i32) {
499 TCGv_i32 t = tcg_temp_new_i32();
500 tcg_gen_sari_i32(t, arg, 31);
501 tcg_gen_xor_i32(t, t, arg);
502 tcg_gen_clzi_i32(t, t, 32);
503 tcg_gen_subi_i32(ret, t, 1);
504 tcg_temp_free_i32(t);
505 } else {
506 gen_helper_clrsb_i32(ret, arg);
507 }
508 }
509
510 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
511 {
512 if (TCG_TARGET_HAS_ctpop_i32) {
513 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
514 } else if (TCG_TARGET_HAS_ctpop_i64) {
515 TCGv_i64 t = tcg_temp_new_i64();
516 tcg_gen_extu_i32_i64(t, arg1);
517 tcg_gen_ctpop_i64(t, t);
518 tcg_gen_extrl_i64_i32(ret, t);
519 tcg_temp_free_i64(t);
520 } else {
521 gen_helper_ctpop_i32(ret, arg1);
522 }
523 }
524
525 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
526 {
527 if (TCG_TARGET_HAS_rot_i32) {
528 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
529 } else {
530 TCGv_i32 t0, t1;
531
532 t0 = tcg_temp_new_i32();
533 t1 = tcg_temp_new_i32();
534 tcg_gen_shl_i32(t0, arg1, arg2);
535 tcg_gen_subfi_i32(t1, 32, arg2);
536 tcg_gen_shr_i32(t1, arg1, t1);
537 tcg_gen_or_i32(ret, t0, t1);
538 tcg_temp_free_i32(t0);
539 tcg_temp_free_i32(t1);
540 }
541 }
542
543 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
544 {
545 tcg_debug_assert(arg2 < 32);
546 /* some cases can be optimized here */
547 if (arg2 == 0) {
548 tcg_gen_mov_i32(ret, arg1);
549 } else if (TCG_TARGET_HAS_rot_i32) {
550 TCGv_i32 t0 = tcg_const_i32(arg2);
551 tcg_gen_rotl_i32(ret, arg1, t0);
552 tcg_temp_free_i32(t0);
553 } else {
554 TCGv_i32 t0, t1;
555 t0 = tcg_temp_new_i32();
556 t1 = tcg_temp_new_i32();
557 tcg_gen_shli_i32(t0, arg1, arg2);
558 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
559 tcg_gen_or_i32(ret, t0, t1);
560 tcg_temp_free_i32(t0);
561 tcg_temp_free_i32(t1);
562 }
563 }
564
565 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
566 {
567 if (TCG_TARGET_HAS_rot_i32) {
568 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
569 } else {
570 TCGv_i32 t0, t1;
571
572 t0 = tcg_temp_new_i32();
573 t1 = tcg_temp_new_i32();
574 tcg_gen_shr_i32(t0, arg1, arg2);
575 tcg_gen_subfi_i32(t1, 32, arg2);
576 tcg_gen_shl_i32(t1, arg1, t1);
577 tcg_gen_or_i32(ret, t0, t1);
578 tcg_temp_free_i32(t0);
579 tcg_temp_free_i32(t1);
580 }
581 }
582
583 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
584 {
585 tcg_debug_assert(arg2 < 32);
586 /* some cases can be optimized here */
587 if (arg2 == 0) {
588 tcg_gen_mov_i32(ret, arg1);
589 } else {
590 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
591 }
592 }
593
594 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
595 unsigned int ofs, unsigned int len)
596 {
597 uint32_t mask;
598 TCGv_i32 t1;
599
600 tcg_debug_assert(ofs < 32);
601 tcg_debug_assert(len > 0);
602 tcg_debug_assert(len <= 32);
603 tcg_debug_assert(ofs + len <= 32);
604
605 if (len == 32) {
606 tcg_gen_mov_i32(ret, arg2);
607 return;
608 }
609 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
610 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
611 return;
612 }
613
614 mask = (1u << len) - 1;
615 t1 = tcg_temp_new_i32();
616
617 if (ofs + len < 32) {
618 tcg_gen_andi_i32(t1, arg2, mask);
619 tcg_gen_shli_i32(t1, t1, ofs);
620 } else {
621 tcg_gen_shli_i32(t1, arg2, ofs);
622 }
623 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
624 tcg_gen_or_i32(ret, ret, t1);
625
626 tcg_temp_free_i32(t1);
627 }
628
629 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
630 unsigned int ofs, unsigned int len)
631 {
632 tcg_debug_assert(ofs < 32);
633 tcg_debug_assert(len > 0);
634 tcg_debug_assert(len <= 32);
635 tcg_debug_assert(ofs + len <= 32);
636
637 if (ofs + len == 32) {
638 tcg_gen_shli_i32(ret, arg, ofs);
639 } else if (ofs == 0) {
640 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
641 } else if (TCG_TARGET_HAS_deposit_i32
642 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
643 TCGv_i32 zero = tcg_const_i32(0);
644 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
645 tcg_temp_free_i32(zero);
646 } else {
647 /* To help two-operand hosts we prefer to zero-extend first,
648 which allows ARG to stay live. */
649 switch (len) {
650 case 16:
651 if (TCG_TARGET_HAS_ext16u_i32) {
652 tcg_gen_ext16u_i32(ret, arg);
653 tcg_gen_shli_i32(ret, ret, ofs);
654 return;
655 }
656 break;
657 case 8:
658 if (TCG_TARGET_HAS_ext8u_i32) {
659 tcg_gen_ext8u_i32(ret, arg);
660 tcg_gen_shli_i32(ret, ret, ofs);
661 return;
662 }
663 break;
664 }
665 /* Otherwise prefer zero-extension over AND for code size. */
666 switch (ofs + len) {
667 case 16:
668 if (TCG_TARGET_HAS_ext16u_i32) {
669 tcg_gen_shli_i32(ret, arg, ofs);
670 tcg_gen_ext16u_i32(ret, ret);
671 return;
672 }
673 break;
674 case 8:
675 if (TCG_TARGET_HAS_ext8u_i32) {
676 tcg_gen_shli_i32(ret, arg, ofs);
677 tcg_gen_ext8u_i32(ret, ret);
678 return;
679 }
680 break;
681 }
682 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
683 tcg_gen_shli_i32(ret, ret, ofs);
684 }
685 }
686
687 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
688 unsigned int ofs, unsigned int len)
689 {
690 tcg_debug_assert(ofs < 32);
691 tcg_debug_assert(len > 0);
692 tcg_debug_assert(len <= 32);
693 tcg_debug_assert(ofs + len <= 32);
694
695 /* Canonicalize certain special cases, even if extract is supported. */
696 if (ofs + len == 32) {
697 tcg_gen_shri_i32(ret, arg, 32 - len);
698 return;
699 }
700 if (ofs == 0) {
701 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
702 return;
703 }
704
705 if (TCG_TARGET_HAS_extract_i32
706 && TCG_TARGET_extract_i32_valid(ofs, len)) {
707 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
708 return;
709 }
710
711 /* Assume that zero-extension, if available, is cheaper than a shift. */
712 switch (ofs + len) {
713 case 16:
714 if (TCG_TARGET_HAS_ext16u_i32) {
715 tcg_gen_ext16u_i32(ret, arg);
716 tcg_gen_shri_i32(ret, ret, ofs);
717 return;
718 }
719 break;
720 case 8:
721 if (TCG_TARGET_HAS_ext8u_i32) {
722 tcg_gen_ext8u_i32(ret, arg);
723 tcg_gen_shri_i32(ret, ret, ofs);
724 return;
725 }
726 break;
727 }
728
729 /* ??? Ideally we'd know what values are available for immediate AND.
730 Assume that 8 bits are available, plus the special case of 16,
731 so that we get ext8u, ext16u. */
732 switch (len) {
733 case 1 ... 8: case 16:
734 tcg_gen_shri_i32(ret, arg, ofs);
735 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
736 break;
737 default:
738 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
739 tcg_gen_shri_i32(ret, ret, 32 - len);
740 break;
741 }
742 }
743
744 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
745 unsigned int ofs, unsigned int len)
746 {
747 tcg_debug_assert(ofs < 32);
748 tcg_debug_assert(len > 0);
749 tcg_debug_assert(len <= 32);
750 tcg_debug_assert(ofs + len <= 32);
751
752 /* Canonicalize certain special cases, even if extract is supported. */
753 if (ofs + len == 32) {
754 tcg_gen_sari_i32(ret, arg, 32 - len);
755 return;
756 }
757 if (ofs == 0) {
758 switch (len) {
759 case 16:
760 tcg_gen_ext16s_i32(ret, arg);
761 return;
762 case 8:
763 tcg_gen_ext8s_i32(ret, arg);
764 return;
765 }
766 }
767
768 if (TCG_TARGET_HAS_sextract_i32
769 && TCG_TARGET_extract_i32_valid(ofs, len)) {
770 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
771 return;
772 }
773
774 /* Assume that sign-extension, if available, is cheaper than a shift. */
775 switch (ofs + len) {
776 case 16:
777 if (TCG_TARGET_HAS_ext16s_i32) {
778 tcg_gen_ext16s_i32(ret, arg);
779 tcg_gen_sari_i32(ret, ret, ofs);
780 return;
781 }
782 break;
783 case 8:
784 if (TCG_TARGET_HAS_ext8s_i32) {
785 tcg_gen_ext8s_i32(ret, arg);
786 tcg_gen_sari_i32(ret, ret, ofs);
787 return;
788 }
789 break;
790 }
791 switch (len) {
792 case 16:
793 if (TCG_TARGET_HAS_ext16s_i32) {
794 tcg_gen_shri_i32(ret, arg, ofs);
795 tcg_gen_ext16s_i32(ret, ret);
796 return;
797 }
798 break;
799 case 8:
800 if (TCG_TARGET_HAS_ext8s_i32) {
801 tcg_gen_shri_i32(ret, arg, ofs);
802 tcg_gen_ext8s_i32(ret, ret);
803 return;
804 }
805 break;
806 }
807
808 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
809 tcg_gen_sari_i32(ret, ret, 32 - len);
810 }
811
812 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
813 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
814 {
815 if (cond == TCG_COND_ALWAYS) {
816 tcg_gen_mov_i32(ret, v1);
817 } else if (cond == TCG_COND_NEVER) {
818 tcg_gen_mov_i32(ret, v2);
819 } else if (TCG_TARGET_HAS_movcond_i32) {
820 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
821 } else {
822 TCGv_i32 t0 = tcg_temp_new_i32();
823 TCGv_i32 t1 = tcg_temp_new_i32();
824 tcg_gen_setcond_i32(cond, t0, c1, c2);
825 tcg_gen_neg_i32(t0, t0);
826 tcg_gen_and_i32(t1, v1, t0);
827 tcg_gen_andc_i32(ret, v2, t0);
828 tcg_gen_or_i32(ret, ret, t1);
829 tcg_temp_free_i32(t0);
830 tcg_temp_free_i32(t1);
831 }
832 }
833
834 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
835 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
836 {
837 if (TCG_TARGET_HAS_add2_i32) {
838 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
839 } else {
840 TCGv_i64 t0 = tcg_temp_new_i64();
841 TCGv_i64 t1 = tcg_temp_new_i64();
842 tcg_gen_concat_i32_i64(t0, al, ah);
843 tcg_gen_concat_i32_i64(t1, bl, bh);
844 tcg_gen_add_i64(t0, t0, t1);
845 tcg_gen_extr_i64_i32(rl, rh, t0);
846 tcg_temp_free_i64(t0);
847 tcg_temp_free_i64(t1);
848 }
849 }
850
851 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
852 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
853 {
854 if (TCG_TARGET_HAS_sub2_i32) {
855 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
856 } else {
857 TCGv_i64 t0 = tcg_temp_new_i64();
858 TCGv_i64 t1 = tcg_temp_new_i64();
859 tcg_gen_concat_i32_i64(t0, al, ah);
860 tcg_gen_concat_i32_i64(t1, bl, bh);
861 tcg_gen_sub_i64(t0, t0, t1);
862 tcg_gen_extr_i64_i32(rl, rh, t0);
863 tcg_temp_free_i64(t0);
864 tcg_temp_free_i64(t1);
865 }
866 }
867
868 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
869 {
870 if (TCG_TARGET_HAS_mulu2_i32) {
871 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
872 } else if (TCG_TARGET_HAS_muluh_i32) {
873 TCGv_i32 t = tcg_temp_new_i32();
874 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
875 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
876 tcg_gen_mov_i32(rl, t);
877 tcg_temp_free_i32(t);
878 } else {
879 TCGv_i64 t0 = tcg_temp_new_i64();
880 TCGv_i64 t1 = tcg_temp_new_i64();
881 tcg_gen_extu_i32_i64(t0, arg1);
882 tcg_gen_extu_i32_i64(t1, arg2);
883 tcg_gen_mul_i64(t0, t0, t1);
884 tcg_gen_extr_i64_i32(rl, rh, t0);
885 tcg_temp_free_i64(t0);
886 tcg_temp_free_i64(t1);
887 }
888 }
889
890 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
891 {
892 if (TCG_TARGET_HAS_muls2_i32) {
893 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
894 } else if (TCG_TARGET_HAS_mulsh_i32) {
895 TCGv_i32 t = tcg_temp_new_i32();
896 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
897 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
898 tcg_gen_mov_i32(rl, t);
899 tcg_temp_free_i32(t);
900 } else if (TCG_TARGET_REG_BITS == 32) {
901 TCGv_i32 t0 = tcg_temp_new_i32();
902 TCGv_i32 t1 = tcg_temp_new_i32();
903 TCGv_i32 t2 = tcg_temp_new_i32();
904 TCGv_i32 t3 = tcg_temp_new_i32();
905 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
906 /* Adjust for negative inputs. */
907 tcg_gen_sari_i32(t2, arg1, 31);
908 tcg_gen_sari_i32(t3, arg2, 31);
909 tcg_gen_and_i32(t2, t2, arg2);
910 tcg_gen_and_i32(t3, t3, arg1);
911 tcg_gen_sub_i32(rh, t1, t2);
912 tcg_gen_sub_i32(rh, rh, t3);
913 tcg_gen_mov_i32(rl, t0);
914 tcg_temp_free_i32(t0);
915 tcg_temp_free_i32(t1);
916 tcg_temp_free_i32(t2);
917 tcg_temp_free_i32(t3);
918 } else {
919 TCGv_i64 t0 = tcg_temp_new_i64();
920 TCGv_i64 t1 = tcg_temp_new_i64();
921 tcg_gen_ext_i32_i64(t0, arg1);
922 tcg_gen_ext_i32_i64(t1, arg2);
923 tcg_gen_mul_i64(t0, t0, t1);
924 tcg_gen_extr_i64_i32(rl, rh, t0);
925 tcg_temp_free_i64(t0);
926 tcg_temp_free_i64(t1);
927 }
928 }
929
930 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
931 {
932 if (TCG_TARGET_REG_BITS == 32) {
933 TCGv_i32 t0 = tcg_temp_new_i32();
934 TCGv_i32 t1 = tcg_temp_new_i32();
935 TCGv_i32 t2 = tcg_temp_new_i32();
936 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
937 /* Adjust for negative input for the signed arg1. */
938 tcg_gen_sari_i32(t2, arg1, 31);
939 tcg_gen_and_i32(t2, t2, arg2);
940 tcg_gen_sub_i32(rh, t1, t2);
941 tcg_gen_mov_i32(rl, t0);
942 tcg_temp_free_i32(t0);
943 tcg_temp_free_i32(t1);
944 tcg_temp_free_i32(t2);
945 } else {
946 TCGv_i64 t0 = tcg_temp_new_i64();
947 TCGv_i64 t1 = tcg_temp_new_i64();
948 tcg_gen_ext_i32_i64(t0, arg1);
949 tcg_gen_extu_i32_i64(t1, arg2);
950 tcg_gen_mul_i64(t0, t0, t1);
951 tcg_gen_extr_i64_i32(rl, rh, t0);
952 tcg_temp_free_i64(t0);
953 tcg_temp_free_i64(t1);
954 }
955 }
956
957 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
958 {
959 if (TCG_TARGET_HAS_ext8s_i32) {
960 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
961 } else {
962 tcg_gen_shli_i32(ret, arg, 24);
963 tcg_gen_sari_i32(ret, ret, 24);
964 }
965 }
966
967 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
968 {
969 if (TCG_TARGET_HAS_ext16s_i32) {
970 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
971 } else {
972 tcg_gen_shli_i32(ret, arg, 16);
973 tcg_gen_sari_i32(ret, ret, 16);
974 }
975 }
976
977 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
978 {
979 if (TCG_TARGET_HAS_ext8u_i32) {
980 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
981 } else {
982 tcg_gen_andi_i32(ret, arg, 0xffu);
983 }
984 }
985
986 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
987 {
988 if (TCG_TARGET_HAS_ext16u_i32) {
989 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
990 } else {
991 tcg_gen_andi_i32(ret, arg, 0xffffu);
992 }
993 }
994
995 /* Note: we assume the two high bytes are set to zero */
996 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
997 {
998 if (TCG_TARGET_HAS_bswap16_i32) {
999 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
1000 } else {
1001 TCGv_i32 t0 = tcg_temp_new_i32();
1002
1003 tcg_gen_ext8u_i32(t0, arg);
1004 tcg_gen_shli_i32(t0, t0, 8);
1005 tcg_gen_shri_i32(ret, arg, 8);
1006 tcg_gen_or_i32(ret, ret, t0);
1007 tcg_temp_free_i32(t0);
1008 }
1009 }
1010
1011 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1012 {
1013 if (TCG_TARGET_HAS_bswap32_i32) {
1014 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1015 } else {
1016 TCGv_i32 t0 = tcg_temp_new_i32();
1017 TCGv_i32 t1 = tcg_temp_new_i32();
1018 TCGv_i32 t2 = tcg_const_i32(0x00ff00ff);
1019
1020 /* arg = abcd */
1021 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */
1022 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */
1023 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */
1024 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */
1025 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */
1026
1027 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */
1028 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */
1029 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */
1030
1031 tcg_temp_free_i32(t0);
1032 tcg_temp_free_i32(t1);
1033 tcg_temp_free_i32(t2);
1034 }
1035 }
1036
1037 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1038 {
1039 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1040 }
1041
1042 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1043 {
1044 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1045 }
1046
1047 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1048 {
1049 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1050 }
1051
1052 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1053 {
1054 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1055 }
1056
1057 /* 64-bit ops */
1058
1059 #if TCG_TARGET_REG_BITS == 32
1060 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1061
1062 void tcg_gen_discard_i64(TCGv_i64 arg)
1063 {
1064 tcg_gen_discard_i32(TCGV_LOW(arg));
1065 tcg_gen_discard_i32(TCGV_HIGH(arg));
1066 }
1067
1068 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1069 {
1070 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1071 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1072 }
1073
1074 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1075 {
1076 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1077 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1078 }
1079
1080 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1081 {
1082 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1083 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1084 }
1085
1086 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1087 {
1088 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1089 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1090 }
1091
1092 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1093 {
1094 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1095 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1096 }
1097
1098 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1099 {
1100 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1101 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1102 }
1103
1104 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1105 {
1106 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1107 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1108 }
1109
1110 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1111 {
1112 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1113 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1114 }
1115
1116 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1117 {
1118 /* Since arg2 and ret have different types,
1119 they cannot be the same temporary */
1120 #ifdef HOST_WORDS_BIGENDIAN
1121 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1122 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1123 #else
1124 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1125 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1126 #endif
1127 }
1128
1129 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1130 {
1131 #ifdef HOST_WORDS_BIGENDIAN
1132 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1133 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1134 #else
1135 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1136 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1137 #endif
1138 }
1139
1140 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1141 {
1142 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1143 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1144 }
1145
1146 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1147 {
1148 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1149 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1150 }
1151
1152 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1153 {
1154 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1155 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1156 }
1157
1158 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1159 {
1160 gen_helper_shl_i64(ret, arg1, arg2);
1161 }
1162
1163 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1164 {
1165 gen_helper_shr_i64(ret, arg1, arg2);
1166 }
1167
1168 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1169 {
1170 gen_helper_sar_i64(ret, arg1, arg2);
1171 }
1172
1173 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1174 {
1175 TCGv_i64 t0;
1176 TCGv_i32 t1;
1177
1178 t0 = tcg_temp_new_i64();
1179 t1 = tcg_temp_new_i32();
1180
1181 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1182 TCGV_LOW(arg1), TCGV_LOW(arg2));
1183
1184 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1185 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1186 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1187 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1188
1189 tcg_gen_mov_i64(ret, t0);
1190 tcg_temp_free_i64(t0);
1191 tcg_temp_free_i32(t1);
1192 }
1193 #endif /* TCG_TARGET_REG_SIZE == 32 */
1194
1195 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1196 {
1197 /* some cases can be optimized here */
1198 if (arg2 == 0) {
1199 tcg_gen_mov_i64(ret, arg1);
1200 } else {
1201 TCGv_i64 t0 = tcg_const_i64(arg2);
1202 tcg_gen_add_i64(ret, arg1, t0);
1203 tcg_temp_free_i64(t0);
1204 }
1205 }
1206
1207 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1208 {
1209 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1210 /* Don't recurse with tcg_gen_neg_i64. */
1211 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1212 } else {
1213 TCGv_i64 t0 = tcg_const_i64(arg1);
1214 tcg_gen_sub_i64(ret, t0, arg2);
1215 tcg_temp_free_i64(t0);
1216 }
1217 }
1218
1219 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1220 {
1221 /* some cases can be optimized here */
1222 if (arg2 == 0) {
1223 tcg_gen_mov_i64(ret, arg1);
1224 } else {
1225 TCGv_i64 t0 = tcg_const_i64(arg2);
1226 tcg_gen_sub_i64(ret, arg1, t0);
1227 tcg_temp_free_i64(t0);
1228 }
1229 }
1230
1231 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1232 {
1233 TCGv_i64 t0;
1234
1235 if (TCG_TARGET_REG_BITS == 32) {
1236 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1237 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1238 return;
1239 }
1240
1241 /* Some cases can be optimized here. */
1242 switch (arg2) {
1243 case 0:
1244 tcg_gen_movi_i64(ret, 0);
1245 return;
1246 case -1:
1247 tcg_gen_mov_i64(ret, arg1);
1248 return;
1249 case 0xff:
1250 /* Don't recurse with tcg_gen_ext8u_i64. */
1251 if (TCG_TARGET_HAS_ext8u_i64) {
1252 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1253 return;
1254 }
1255 break;
1256 case 0xffff:
1257 if (TCG_TARGET_HAS_ext16u_i64) {
1258 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1259 return;
1260 }
1261 break;
1262 case 0xffffffffu:
1263 if (TCG_TARGET_HAS_ext32u_i64) {
1264 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1265 return;
1266 }
1267 break;
1268 }
1269 t0 = tcg_const_i64(arg2);
1270 tcg_gen_and_i64(ret, arg1, t0);
1271 tcg_temp_free_i64(t0);
1272 }
1273
1274 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1275 {
1276 if (TCG_TARGET_REG_BITS == 32) {
1277 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1278 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1279 return;
1280 }
1281 /* Some cases can be optimized here. */
1282 if (arg2 == -1) {
1283 tcg_gen_movi_i64(ret, -1);
1284 } else if (arg2 == 0) {
1285 tcg_gen_mov_i64(ret, arg1);
1286 } else {
1287 TCGv_i64 t0 = tcg_const_i64(arg2);
1288 tcg_gen_or_i64(ret, arg1, t0);
1289 tcg_temp_free_i64(t0);
1290 }
1291 }
1292
1293 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1294 {
1295 if (TCG_TARGET_REG_BITS == 32) {
1296 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1297 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1298 return;
1299 }
1300 /* Some cases can be optimized here. */
1301 if (arg2 == 0) {
1302 tcg_gen_mov_i64(ret, arg1);
1303 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1304 /* Don't recurse with tcg_gen_not_i64. */
1305 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1306 } else {
1307 TCGv_i64 t0 = tcg_const_i64(arg2);
1308 tcg_gen_xor_i64(ret, arg1, t0);
1309 tcg_temp_free_i64(t0);
1310 }
1311 }
1312
1313 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1314 unsigned c, bool right, bool arith)
1315 {
1316 tcg_debug_assert(c < 64);
1317 if (c == 0) {
1318 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1319 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1320 } else if (c >= 32) {
1321 c -= 32;
1322 if (right) {
1323 if (arith) {
1324 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1325 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1326 } else {
1327 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1328 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1329 }
1330 } else {
1331 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1332 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1333 }
1334 } else {
1335 TCGv_i32 t0, t1;
1336
1337 t0 = tcg_temp_new_i32();
1338 t1 = tcg_temp_new_i32();
1339 if (right) {
1340 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1341 if (arith) {
1342 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1343 } else {
1344 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1345 }
1346 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1347 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1348 tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1349 } else {
1350 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1351 /* Note: ret can be the same as arg1, so we use t1 */
1352 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1353 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1354 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1355 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1356 }
1357 tcg_temp_free_i32(t0);
1358 tcg_temp_free_i32(t1);
1359 }
1360 }
1361
1362 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1363 {
1364 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1365 if (TCG_TARGET_REG_BITS == 32) {
1366 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1367 } else if (arg2 == 0) {
1368 tcg_gen_mov_i64(ret, arg1);
1369 } else {
1370 TCGv_i64 t0 = tcg_const_i64(arg2);
1371 tcg_gen_shl_i64(ret, arg1, t0);
1372 tcg_temp_free_i64(t0);
1373 }
1374 }
1375
1376 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1377 {
1378 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1379 if (TCG_TARGET_REG_BITS == 32) {
1380 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1381 } else if (arg2 == 0) {
1382 tcg_gen_mov_i64(ret, arg1);
1383 } else {
1384 TCGv_i64 t0 = tcg_const_i64(arg2);
1385 tcg_gen_shr_i64(ret, arg1, t0);
1386 tcg_temp_free_i64(t0);
1387 }
1388 }
1389
1390 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1391 {
1392 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1393 if (TCG_TARGET_REG_BITS == 32) {
1394 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1395 } else if (arg2 == 0) {
1396 tcg_gen_mov_i64(ret, arg1);
1397 } else {
1398 TCGv_i64 t0 = tcg_const_i64(arg2);
1399 tcg_gen_sar_i64(ret, arg1, t0);
1400 tcg_temp_free_i64(t0);
1401 }
1402 }
1403
1404 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1405 {
1406 if (cond == TCG_COND_ALWAYS) {
1407 tcg_gen_br(l);
1408 } else if (cond != TCG_COND_NEVER) {
1409 l->refs++;
1410 if (TCG_TARGET_REG_BITS == 32) {
1411 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1412 TCGV_HIGH(arg1), TCGV_LOW(arg2),
1413 TCGV_HIGH(arg2), cond, label_arg(l));
1414 } else {
1415 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1416 label_arg(l));
1417 }
1418 }
1419 }
1420
1421 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1422 {
1423 if (cond == TCG_COND_ALWAYS) {
1424 tcg_gen_br(l);
1425 } else if (cond != TCG_COND_NEVER) {
1426 TCGv_i64 t0 = tcg_const_i64(arg2);
1427 tcg_gen_brcond_i64(cond, arg1, t0, l);
1428 tcg_temp_free_i64(t0);
1429 }
1430 }
1431
1432 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1433 TCGv_i64 arg1, TCGv_i64 arg2)
1434 {
1435 if (cond == TCG_COND_ALWAYS) {
1436 tcg_gen_movi_i64(ret, 1);
1437 } else if (cond == TCG_COND_NEVER) {
1438 tcg_gen_movi_i64(ret, 0);
1439 } else {
1440 if (TCG_TARGET_REG_BITS == 32) {
1441 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1442 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1443 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1444 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1445 } else {
1446 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1447 }
1448 }
1449 }
1450
1451 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1452 TCGv_i64 arg1, int64_t arg2)
1453 {
1454 TCGv_i64 t0 = tcg_const_i64(arg2);
1455 tcg_gen_setcond_i64(cond, ret, arg1, t0);
1456 tcg_temp_free_i64(t0);
1457 }
1458
1459 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1460 {
1461 if (arg2 == 0) {
1462 tcg_gen_movi_i64(ret, 0);
1463 } else if (is_power_of_2(arg2)) {
1464 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1465 } else {
1466 TCGv_i64 t0 = tcg_const_i64(arg2);
1467 tcg_gen_mul_i64(ret, arg1, t0);
1468 tcg_temp_free_i64(t0);
1469 }
1470 }
1471
1472 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1473 {
1474 if (TCG_TARGET_HAS_div_i64) {
1475 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1476 } else if (TCG_TARGET_HAS_div2_i64) {
1477 TCGv_i64 t0 = tcg_temp_new_i64();
1478 tcg_gen_sari_i64(t0, arg1, 63);
1479 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1480 tcg_temp_free_i64(t0);
1481 } else {
1482 gen_helper_div_i64(ret, arg1, arg2);
1483 }
1484 }
1485
1486 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1487 {
1488 if (TCG_TARGET_HAS_rem_i64) {
1489 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1490 } else if (TCG_TARGET_HAS_div_i64) {
1491 TCGv_i64 t0 = tcg_temp_new_i64();
1492 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1493 tcg_gen_mul_i64(t0, t0, arg2);
1494 tcg_gen_sub_i64(ret, arg1, t0);
1495 tcg_temp_free_i64(t0);
1496 } else if (TCG_TARGET_HAS_div2_i64) {
1497 TCGv_i64 t0 = tcg_temp_new_i64();
1498 tcg_gen_sari_i64(t0, arg1, 63);
1499 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1500 tcg_temp_free_i64(t0);
1501 } else {
1502 gen_helper_rem_i64(ret, arg1, arg2);
1503 }
1504 }
1505
1506 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1507 {
1508 if (TCG_TARGET_HAS_div_i64) {
1509 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1510 } else if (TCG_TARGET_HAS_div2_i64) {
1511 TCGv_i64 t0 = tcg_temp_new_i64();
1512 tcg_gen_movi_i64(t0, 0);
1513 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1514 tcg_temp_free_i64(t0);
1515 } else {
1516 gen_helper_divu_i64(ret, arg1, arg2);
1517 }
1518 }
1519
1520 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1521 {
1522 if (TCG_TARGET_HAS_rem_i64) {
1523 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1524 } else if (TCG_TARGET_HAS_div_i64) {
1525 TCGv_i64 t0 = tcg_temp_new_i64();
1526 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1527 tcg_gen_mul_i64(t0, t0, arg2);
1528 tcg_gen_sub_i64(ret, arg1, t0);
1529 tcg_temp_free_i64(t0);
1530 } else if (TCG_TARGET_HAS_div2_i64) {
1531 TCGv_i64 t0 = tcg_temp_new_i64();
1532 tcg_gen_movi_i64(t0, 0);
1533 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1534 tcg_temp_free_i64(t0);
1535 } else {
1536 gen_helper_remu_i64(ret, arg1, arg2);
1537 }
1538 }
1539
1540 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1541 {
1542 if (TCG_TARGET_REG_BITS == 32) {
1543 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1544 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1545 } else if (TCG_TARGET_HAS_ext8s_i64) {
1546 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1547 } else {
1548 tcg_gen_shli_i64(ret, arg, 56);
1549 tcg_gen_sari_i64(ret, ret, 56);
1550 }
1551 }
1552
1553 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1554 {
1555 if (TCG_TARGET_REG_BITS == 32) {
1556 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1557 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1558 } else if (TCG_TARGET_HAS_ext16s_i64) {
1559 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1560 } else {
1561 tcg_gen_shli_i64(ret, arg, 48);
1562 tcg_gen_sari_i64(ret, ret, 48);
1563 }
1564 }
1565
1566 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1567 {
1568 if (TCG_TARGET_REG_BITS == 32) {
1569 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1570 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1571 } else if (TCG_TARGET_HAS_ext32s_i64) {
1572 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1573 } else {
1574 tcg_gen_shli_i64(ret, arg, 32);
1575 tcg_gen_sari_i64(ret, ret, 32);
1576 }
1577 }
1578
1579 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1580 {
1581 if (TCG_TARGET_REG_BITS == 32) {
1582 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1583 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1584 } else if (TCG_TARGET_HAS_ext8u_i64) {
1585 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1586 } else {
1587 tcg_gen_andi_i64(ret, arg, 0xffu);
1588 }
1589 }
1590
1591 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1592 {
1593 if (TCG_TARGET_REG_BITS == 32) {
1594 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1595 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1596 } else if (TCG_TARGET_HAS_ext16u_i64) {
1597 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1598 } else {
1599 tcg_gen_andi_i64(ret, arg, 0xffffu);
1600 }
1601 }
1602
1603 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1604 {
1605 if (TCG_TARGET_REG_BITS == 32) {
1606 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1607 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1608 } else if (TCG_TARGET_HAS_ext32u_i64) {
1609 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1610 } else {
1611 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1612 }
1613 }
1614
1615 /* Note: we assume the six high bytes are set to zero */
1616 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1617 {
1618 if (TCG_TARGET_REG_BITS == 32) {
1619 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1620 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1621 } else if (TCG_TARGET_HAS_bswap16_i64) {
1622 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1623 } else {
1624 TCGv_i64 t0 = tcg_temp_new_i64();
1625
1626 tcg_gen_ext8u_i64(t0, arg);
1627 tcg_gen_shli_i64(t0, t0, 8);
1628 tcg_gen_shri_i64(ret, arg, 8);
1629 tcg_gen_or_i64(ret, ret, t0);
1630 tcg_temp_free_i64(t0);
1631 }
1632 }
1633
1634 /* Note: we assume the four high bytes are set to zero */
1635 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1636 {
1637 if (TCG_TARGET_REG_BITS == 32) {
1638 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1639 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1640 } else if (TCG_TARGET_HAS_bswap32_i64) {
1641 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1642 } else {
1643 TCGv_i64 t0 = tcg_temp_new_i64();
1644 TCGv_i64 t1 = tcg_temp_new_i64();
1645 TCGv_i64 t2 = tcg_const_i64(0x00ff00ff);
1646
1647 /* arg = ....abcd */
1648 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .....abc */
1649 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */
1650 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */
1651 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */
1652 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */
1653
1654 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1655 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1656 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1657 tcg_gen_or_i64(ret, t0, t1); /* ret = ....dcba */
1658
1659 tcg_temp_free_i64(t0);
1660 tcg_temp_free_i64(t1);
1661 tcg_temp_free_i64(t2);
1662 }
1663 }
1664
1665 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1666 {
1667 if (TCG_TARGET_REG_BITS == 32) {
1668 TCGv_i32 t0, t1;
1669 t0 = tcg_temp_new_i32();
1670 t1 = tcg_temp_new_i32();
1671
1672 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1673 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1674 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1675 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1676 tcg_temp_free_i32(t0);
1677 tcg_temp_free_i32(t1);
1678 } else if (TCG_TARGET_HAS_bswap64_i64) {
1679 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1680 } else {
1681 TCGv_i64 t0 = tcg_temp_new_i64();
1682 TCGv_i64 t1 = tcg_temp_new_i64();
1683 TCGv_i64 t2 = tcg_temp_new_i64();
1684
1685 /* arg = abcdefgh */
1686 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1687 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */
1688 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */
1689 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */
1690 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */
1691 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */
1692
1693 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1694 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */
1695 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */
1696 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */
1697 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */
1698 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */
1699
1700 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */
1701 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */
1702 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */
1703
1704 tcg_temp_free_i64(t0);
1705 tcg_temp_free_i64(t1);
1706 tcg_temp_free_i64(t2);
1707 }
1708 }
1709
1710 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1711 {
1712 if (TCG_TARGET_REG_BITS == 32) {
1713 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1714 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1715 } else if (TCG_TARGET_HAS_not_i64) {
1716 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1717 } else {
1718 tcg_gen_xori_i64(ret, arg, -1);
1719 }
1720 }
1721
1722 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1723 {
1724 if (TCG_TARGET_REG_BITS == 32) {
1725 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1726 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1727 } else if (TCG_TARGET_HAS_andc_i64) {
1728 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1729 } else {
1730 TCGv_i64 t0 = tcg_temp_new_i64();
1731 tcg_gen_not_i64(t0, arg2);
1732 tcg_gen_and_i64(ret, arg1, t0);
1733 tcg_temp_free_i64(t0);
1734 }
1735 }
1736
1737 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1738 {
1739 if (TCG_TARGET_REG_BITS == 32) {
1740 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1741 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1742 } else if (TCG_TARGET_HAS_eqv_i64) {
1743 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1744 } else {
1745 tcg_gen_xor_i64(ret, arg1, arg2);
1746 tcg_gen_not_i64(ret, ret);
1747 }
1748 }
1749
1750 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1751 {
1752 if (TCG_TARGET_REG_BITS == 32) {
1753 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1754 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1755 } else if (TCG_TARGET_HAS_nand_i64) {
1756 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1757 } else {
1758 tcg_gen_and_i64(ret, arg1, arg2);
1759 tcg_gen_not_i64(ret, ret);
1760 }
1761 }
1762
1763 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1764 {
1765 if (TCG_TARGET_REG_BITS == 32) {
1766 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1767 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1768 } else if (TCG_TARGET_HAS_nor_i64) {
1769 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1770 } else {
1771 tcg_gen_or_i64(ret, arg1, arg2);
1772 tcg_gen_not_i64(ret, ret);
1773 }
1774 }
1775
1776 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1777 {
1778 if (TCG_TARGET_REG_BITS == 32) {
1779 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1780 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1781 } else if (TCG_TARGET_HAS_orc_i64) {
1782 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1783 } else {
1784 TCGv_i64 t0 = tcg_temp_new_i64();
1785 tcg_gen_not_i64(t0, arg2);
1786 tcg_gen_or_i64(ret, arg1, t0);
1787 tcg_temp_free_i64(t0);
1788 }
1789 }
1790
1791 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1792 {
1793 if (TCG_TARGET_HAS_clz_i64) {
1794 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1795 } else {
1796 gen_helper_clz_i64(ret, arg1, arg2);
1797 }
1798 }
1799
1800 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1801 {
1802 if (TCG_TARGET_REG_BITS == 32
1803 && TCG_TARGET_HAS_clz_i32
1804 && arg2 <= 0xffffffffu) {
1805 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1806 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1807 tcg_gen_addi_i32(t, t, 32);
1808 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1809 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1810 tcg_temp_free_i32(t);
1811 } else {
1812 TCGv_i64 t = tcg_const_i64(arg2);
1813 tcg_gen_clz_i64(ret, arg1, t);
1814 tcg_temp_free_i64(t);
1815 }
1816 }
1817
1818 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1819 {
1820 if (TCG_TARGET_HAS_ctz_i64) {
1821 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1822 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1823 TCGv_i64 z, t = tcg_temp_new_i64();
1824
1825 if (TCG_TARGET_HAS_ctpop_i64) {
1826 tcg_gen_subi_i64(t, arg1, 1);
1827 tcg_gen_andc_i64(t, t, arg1);
1828 tcg_gen_ctpop_i64(t, t);
1829 } else {
1830 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1831 tcg_gen_neg_i64(t, arg1);
1832 tcg_gen_and_i64(t, t, arg1);
1833 tcg_gen_clzi_i64(t, t, 64);
1834 tcg_gen_xori_i64(t, t, 63);
1835 }
1836 z = tcg_const_i64(0);
1837 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1838 tcg_temp_free_i64(t);
1839 tcg_temp_free_i64(z);
1840 } else {
1841 gen_helper_ctz_i64(ret, arg1, arg2);
1842 }
1843 }
1844
1845 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1846 {
1847 if (TCG_TARGET_REG_BITS == 32
1848 && TCG_TARGET_HAS_ctz_i32
1849 && arg2 <= 0xffffffffu) {
1850 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1851 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1852 tcg_gen_addi_i32(t32, t32, 32);
1853 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1854 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1855 tcg_temp_free_i32(t32);
1856 } else if (!TCG_TARGET_HAS_ctz_i64
1857 && TCG_TARGET_HAS_ctpop_i64
1858 && arg2 == 64) {
1859 /* This equivalence has the advantage of not requiring a fixup. */
1860 TCGv_i64 t = tcg_temp_new_i64();
1861 tcg_gen_subi_i64(t, arg1, 1);
1862 tcg_gen_andc_i64(t, t, arg1);
1863 tcg_gen_ctpop_i64(ret, t);
1864 tcg_temp_free_i64(t);
1865 } else {
1866 TCGv_i64 t64 = tcg_const_i64(arg2);
1867 tcg_gen_ctz_i64(ret, arg1, t64);
1868 tcg_temp_free_i64(t64);
1869 }
1870 }
1871
1872 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1873 {
1874 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1875 TCGv_i64 t = tcg_temp_new_i64();
1876 tcg_gen_sari_i64(t, arg, 63);
1877 tcg_gen_xor_i64(t, t, arg);
1878 tcg_gen_clzi_i64(t, t, 64);
1879 tcg_gen_subi_i64(ret, t, 1);
1880 tcg_temp_free_i64(t);
1881 } else {
1882 gen_helper_clrsb_i64(ret, arg);
1883 }
1884 }
1885
1886 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1887 {
1888 if (TCG_TARGET_HAS_ctpop_i64) {
1889 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1890 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1891 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1892 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1893 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1894 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1895 } else {
1896 gen_helper_ctpop_i64(ret, arg1);
1897 }
1898 }
1899
1900 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1901 {
1902 if (TCG_TARGET_HAS_rot_i64) {
1903 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1904 } else {
1905 TCGv_i64 t0, t1;
1906 t0 = tcg_temp_new_i64();
1907 t1 = tcg_temp_new_i64();
1908 tcg_gen_shl_i64(t0, arg1, arg2);
1909 tcg_gen_subfi_i64(t1, 64, arg2);
1910 tcg_gen_shr_i64(t1, arg1, t1);
1911 tcg_gen_or_i64(ret, t0, t1);
1912 tcg_temp_free_i64(t0);
1913 tcg_temp_free_i64(t1);
1914 }
1915 }
1916
1917 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1918 {
1919 tcg_debug_assert(arg2 < 64);
1920 /* some cases can be optimized here */
1921 if (arg2 == 0) {
1922 tcg_gen_mov_i64(ret, arg1);
1923 } else if (TCG_TARGET_HAS_rot_i64) {
1924 TCGv_i64 t0 = tcg_const_i64(arg2);
1925 tcg_gen_rotl_i64(ret, arg1, t0);
1926 tcg_temp_free_i64(t0);
1927 } else {
1928 TCGv_i64 t0, t1;
1929 t0 = tcg_temp_new_i64();
1930 t1 = tcg_temp_new_i64();
1931 tcg_gen_shli_i64(t0, arg1, arg2);
1932 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1933 tcg_gen_or_i64(ret, t0, t1);
1934 tcg_temp_free_i64(t0);
1935 tcg_temp_free_i64(t1);
1936 }
1937 }
1938
1939 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1940 {
1941 if (TCG_TARGET_HAS_rot_i64) {
1942 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1943 } else {
1944 TCGv_i64 t0, t1;
1945 t0 = tcg_temp_new_i64();
1946 t1 = tcg_temp_new_i64();
1947 tcg_gen_shr_i64(t0, arg1, arg2);
1948 tcg_gen_subfi_i64(t1, 64, arg2);
1949 tcg_gen_shl_i64(t1, arg1, t1);
1950 tcg_gen_or_i64(ret, t0, t1);
1951 tcg_temp_free_i64(t0);
1952 tcg_temp_free_i64(t1);
1953 }
1954 }
1955
1956 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1957 {
1958 tcg_debug_assert(arg2 < 64);
1959 /* some cases can be optimized here */
1960 if (arg2 == 0) {
1961 tcg_gen_mov_i64(ret, arg1);
1962 } else {
1963 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1964 }
1965 }
1966
1967 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1968 unsigned int ofs, unsigned int len)
1969 {
1970 uint64_t mask;
1971 TCGv_i64 t1;
1972
1973 tcg_debug_assert(ofs < 64);
1974 tcg_debug_assert(len > 0);
1975 tcg_debug_assert(len <= 64);
1976 tcg_debug_assert(ofs + len <= 64);
1977
1978 if (len == 64) {
1979 tcg_gen_mov_i64(ret, arg2);
1980 return;
1981 }
1982 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1983 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1984 return;
1985 }
1986
1987 if (TCG_TARGET_REG_BITS == 32) {
1988 if (ofs >= 32) {
1989 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1990 TCGV_LOW(arg2), ofs - 32, len);
1991 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1992 return;
1993 }
1994 if (ofs + len <= 32) {
1995 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
1996 TCGV_LOW(arg2), ofs, len);
1997 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1998 return;
1999 }
2000 }
2001
2002 mask = (1ull << len) - 1;
2003 t1 = tcg_temp_new_i64();
2004
2005 if (ofs + len < 64) {
2006 tcg_gen_andi_i64(t1, arg2, mask);
2007 tcg_gen_shli_i64(t1, t1, ofs);
2008 } else {
2009 tcg_gen_shli_i64(t1, arg2, ofs);
2010 }
2011 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2012 tcg_gen_or_i64(ret, ret, t1);
2013
2014 tcg_temp_free_i64(t1);
2015 }
2016
2017 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2018 unsigned int ofs, unsigned int len)
2019 {
2020 tcg_debug_assert(ofs < 64);
2021 tcg_debug_assert(len > 0);
2022 tcg_debug_assert(len <= 64);
2023 tcg_debug_assert(ofs + len <= 64);
2024
2025 if (ofs + len == 64) {
2026 tcg_gen_shli_i64(ret, arg, ofs);
2027 } else if (ofs == 0) {
2028 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2029 } else if (TCG_TARGET_HAS_deposit_i64
2030 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2031 TCGv_i64 zero = tcg_const_i64(0);
2032 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2033 tcg_temp_free_i64(zero);
2034 } else {
2035 if (TCG_TARGET_REG_BITS == 32) {
2036 if (ofs >= 32) {
2037 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2038 ofs - 32, len);
2039 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2040 return;
2041 }
2042 if (ofs + len <= 32) {
2043 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2044 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2045 return;
2046 }
2047 }
2048 /* To help two-operand hosts we prefer to zero-extend first,
2049 which allows ARG to stay live. */
2050 switch (len) {
2051 case 32:
2052 if (TCG_TARGET_HAS_ext32u_i64) {
2053 tcg_gen_ext32u_i64(ret, arg);
2054 tcg_gen_shli_i64(ret, ret, ofs);
2055 return;
2056 }
2057 break;
2058 case 16:
2059 if (TCG_TARGET_HAS_ext16u_i64) {
2060 tcg_gen_ext16u_i64(ret, arg);
2061 tcg_gen_shli_i64(ret, ret, ofs);
2062 return;
2063 }
2064 break;
2065 case 8:
2066 if (TCG_TARGET_HAS_ext8u_i64) {
2067 tcg_gen_ext8u_i64(ret, arg);
2068 tcg_gen_shli_i64(ret, ret, ofs);
2069 return;
2070 }
2071 break;
2072 }
2073 /* Otherwise prefer zero-extension over AND for code size. */
2074 switch (ofs + len) {
2075 case 32:
2076 if (TCG_TARGET_HAS_ext32u_i64) {
2077 tcg_gen_shli_i64(ret, arg, ofs);
2078 tcg_gen_ext32u_i64(ret, ret);
2079 return;
2080 }
2081 break;
2082 case 16:
2083 if (TCG_TARGET_HAS_ext16u_i64) {
2084 tcg_gen_shli_i64(ret, arg, ofs);
2085 tcg_gen_ext16u_i64(ret, ret);
2086 return;
2087 }
2088 break;
2089 case 8:
2090 if (TCG_TARGET_HAS_ext8u_i64) {
2091 tcg_gen_shli_i64(ret, arg, ofs);
2092 tcg_gen_ext8u_i64(ret, ret);
2093 return;
2094 }
2095 break;
2096 }
2097 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2098 tcg_gen_shli_i64(ret, ret, ofs);
2099 }
2100 }
2101
2102 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2103 unsigned int ofs, unsigned int len)
2104 {
2105 tcg_debug_assert(ofs < 64);
2106 tcg_debug_assert(len > 0);
2107 tcg_debug_assert(len <= 64);
2108 tcg_debug_assert(ofs + len <= 64);
2109
2110 /* Canonicalize certain special cases, even if extract is supported. */
2111 if (ofs + len == 64) {
2112 tcg_gen_shri_i64(ret, arg, 64 - len);
2113 return;
2114 }
2115 if (ofs == 0) {
2116 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2117 return;
2118 }
2119
2120 if (TCG_TARGET_REG_BITS == 32) {
2121 /* Look for a 32-bit extract within one of the two words. */
2122 if (ofs >= 32) {
2123 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2124 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2125 return;
2126 }
2127 if (ofs + len <= 32) {
2128 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2129 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2130 return;
2131 }
2132 /* The field is split across two words. One double-word
2133 shift is better than two double-word shifts. */
2134 goto do_shift_and;
2135 }
2136
2137 if (TCG_TARGET_HAS_extract_i64
2138 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2139 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2140 return;
2141 }
2142
2143 /* Assume that zero-extension, if available, is cheaper than a shift. */
2144 switch (ofs + len) {
2145 case 32:
2146 if (TCG_TARGET_HAS_ext32u_i64) {
2147 tcg_gen_ext32u_i64(ret, arg);
2148 tcg_gen_shri_i64(ret, ret, ofs);
2149 return;
2150 }
2151 break;
2152 case 16:
2153 if (TCG_TARGET_HAS_ext16u_i64) {
2154 tcg_gen_ext16u_i64(ret, arg);
2155 tcg_gen_shri_i64(ret, ret, ofs);
2156 return;
2157 }
2158 break;
2159 case 8:
2160 if (TCG_TARGET_HAS_ext8u_i64) {
2161 tcg_gen_ext8u_i64(ret, arg);
2162 tcg_gen_shri_i64(ret, ret, ofs);
2163 return;
2164 }
2165 break;
2166 }
2167
2168 /* ??? Ideally we'd know what values are available for immediate AND.
2169 Assume that 8 bits are available, plus the special cases of 16 and 32,
2170 so that we get ext8u, ext16u, and ext32u. */
2171 switch (len) {
2172 case 1 ... 8: case 16: case 32:
2173 do_shift_and:
2174 tcg_gen_shri_i64(ret, arg, ofs);
2175 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2176 break;
2177 default:
2178 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2179 tcg_gen_shri_i64(ret, ret, 64 - len);
2180 break;
2181 }
2182 }
2183
2184 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2185 unsigned int ofs, unsigned int len)
2186 {
2187 tcg_debug_assert(ofs < 64);
2188 tcg_debug_assert(len > 0);
2189 tcg_debug_assert(len <= 64);
2190 tcg_debug_assert(ofs + len <= 64);
2191
2192 /* Canonicalize certain special cases, even if sextract is supported. */
2193 if (ofs + len == 64) {
2194 tcg_gen_sari_i64(ret, arg, 64 - len);
2195 return;
2196 }
2197 if (ofs == 0) {
2198 switch (len) {
2199 case 32:
2200 tcg_gen_ext32s_i64(ret, arg);
2201 return;
2202 case 16:
2203 tcg_gen_ext16s_i64(ret, arg);
2204 return;
2205 case 8:
2206 tcg_gen_ext8s_i64(ret, arg);
2207 return;
2208 }
2209 }
2210
2211 if (TCG_TARGET_REG_BITS == 32) {
2212 /* Look for a 32-bit extract within one of the two words. */
2213 if (ofs >= 32) {
2214 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2215 } else if (ofs + len <= 32) {
2216 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2217 } else if (ofs == 0) {
2218 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2219 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2220 return;
2221 } else if (len > 32) {
2222 TCGv_i32 t = tcg_temp_new_i32();
2223 /* Extract the bits for the high word normally. */
2224 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2225 /* Shift the field down for the low part. */
2226 tcg_gen_shri_i64(ret, arg, ofs);
2227 /* Overwrite the shift into the high part. */
2228 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2229 tcg_temp_free_i32(t);
2230 return;
2231 } else {
2232 /* Shift the field down for the low part, such that the
2233 field sits at the MSB. */
2234 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2235 /* Shift the field down from the MSB, sign extending. */
2236 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2237 }
2238 /* Sign-extend the field from 32 bits. */
2239 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2240 return;
2241 }
2242
2243 if (TCG_TARGET_HAS_sextract_i64
2244 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2245 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2246 return;
2247 }
2248
2249 /* Assume that sign-extension, if available, is cheaper than a shift. */
2250 switch (ofs + len) {
2251 case 32:
2252 if (TCG_TARGET_HAS_ext32s_i64) {
2253 tcg_gen_ext32s_i64(ret, arg);
2254 tcg_gen_sari_i64(ret, ret, ofs);
2255 return;
2256 }
2257 break;
2258 case 16:
2259 if (TCG_TARGET_HAS_ext16s_i64) {
2260 tcg_gen_ext16s_i64(ret, arg);
2261 tcg_gen_sari_i64(ret, ret, ofs);
2262 return;
2263 }
2264 break;
2265 case 8:
2266 if (TCG_TARGET_HAS_ext8s_i64) {
2267 tcg_gen_ext8s_i64(ret, arg);
2268 tcg_gen_sari_i64(ret, ret, ofs);
2269 return;
2270 }
2271 break;
2272 }
2273 switch (len) {
2274 case 32:
2275 if (TCG_TARGET_HAS_ext32s_i64) {
2276 tcg_gen_shri_i64(ret, arg, ofs);
2277 tcg_gen_ext32s_i64(ret, ret);
2278 return;
2279 }
2280 break;
2281 case 16:
2282 if (TCG_TARGET_HAS_ext16s_i64) {
2283 tcg_gen_shri_i64(ret, arg, ofs);
2284 tcg_gen_ext16s_i64(ret, ret);
2285 return;
2286 }
2287 break;
2288 case 8:
2289 if (TCG_TARGET_HAS_ext8s_i64) {
2290 tcg_gen_shri_i64(ret, arg, ofs);
2291 tcg_gen_ext8s_i64(ret, ret);
2292 return;
2293 }
2294 break;
2295 }
2296 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2297 tcg_gen_sari_i64(ret, ret, 64 - len);
2298 }
2299
2300 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2301 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2302 {
2303 if (cond == TCG_COND_ALWAYS) {
2304 tcg_gen_mov_i64(ret, v1);
2305 } else if (cond == TCG_COND_NEVER) {
2306 tcg_gen_mov_i64(ret, v2);
2307 } else if (TCG_TARGET_REG_BITS == 32) {
2308 TCGv_i32 t0 = tcg_temp_new_i32();
2309 TCGv_i32 t1 = tcg_temp_new_i32();
2310 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2311 TCGV_LOW(c1), TCGV_HIGH(c1),
2312 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2313
2314 if (TCG_TARGET_HAS_movcond_i32) {
2315 tcg_gen_movi_i32(t1, 0);
2316 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2317 TCGV_LOW(v1), TCGV_LOW(v2));
2318 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2319 TCGV_HIGH(v1), TCGV_HIGH(v2));
2320 } else {
2321 tcg_gen_neg_i32(t0, t0);
2322
2323 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2324 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2325 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2326
2327 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2328 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2329 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2330 }
2331 tcg_temp_free_i32(t0);
2332 tcg_temp_free_i32(t1);
2333 } else if (TCG_TARGET_HAS_movcond_i64) {
2334 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2335 } else {
2336 TCGv_i64 t0 = tcg_temp_new_i64();
2337 TCGv_i64 t1 = tcg_temp_new_i64();
2338 tcg_gen_setcond_i64(cond, t0, c1, c2);
2339 tcg_gen_neg_i64(t0, t0);
2340 tcg_gen_and_i64(t1, v1, t0);
2341 tcg_gen_andc_i64(ret, v2, t0);
2342 tcg_gen_or_i64(ret, ret, t1);
2343 tcg_temp_free_i64(t0);
2344 tcg_temp_free_i64(t1);
2345 }
2346 }
2347
2348 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2349 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2350 {
2351 if (TCG_TARGET_HAS_add2_i64) {
2352 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2353 } else {
2354 TCGv_i64 t0 = tcg_temp_new_i64();
2355 TCGv_i64 t1 = tcg_temp_new_i64();
2356 tcg_gen_add_i64(t0, al, bl);
2357 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2358 tcg_gen_add_i64(rh, ah, bh);
2359 tcg_gen_add_i64(rh, rh, t1);
2360 tcg_gen_mov_i64(rl, t0);
2361 tcg_temp_free_i64(t0);
2362 tcg_temp_free_i64(t1);
2363 }
2364 }
2365
2366 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2367 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2368 {
2369 if (TCG_TARGET_HAS_sub2_i64) {
2370 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2371 } else {
2372 TCGv_i64 t0 = tcg_temp_new_i64();
2373 TCGv_i64 t1 = tcg_temp_new_i64();
2374 tcg_gen_sub_i64(t0, al, bl);
2375 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2376 tcg_gen_sub_i64(rh, ah, bh);
2377 tcg_gen_sub_i64(rh, rh, t1);
2378 tcg_gen_mov_i64(rl, t0);
2379 tcg_temp_free_i64(t0);
2380 tcg_temp_free_i64(t1);
2381 }
2382 }
2383
2384 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2385 {
2386 if (TCG_TARGET_HAS_mulu2_i64) {
2387 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2388 } else if (TCG_TARGET_HAS_muluh_i64) {
2389 TCGv_i64 t = tcg_temp_new_i64();
2390 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2391 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2392 tcg_gen_mov_i64(rl, t);
2393 tcg_temp_free_i64(t);
2394 } else {
2395 TCGv_i64 t0 = tcg_temp_new_i64();
2396 tcg_gen_mul_i64(t0, arg1, arg2);
2397 gen_helper_muluh_i64(rh, arg1, arg2);
2398 tcg_gen_mov_i64(rl, t0);
2399 tcg_temp_free_i64(t0);
2400 }
2401 }
2402
2403 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2404 {
2405 if (TCG_TARGET_HAS_muls2_i64) {
2406 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2407 } else if (TCG_TARGET_HAS_mulsh_i64) {
2408 TCGv_i64 t = tcg_temp_new_i64();
2409 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2410 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2411 tcg_gen_mov_i64(rl, t);
2412 tcg_temp_free_i64(t);
2413 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2414 TCGv_i64 t0 = tcg_temp_new_i64();
2415 TCGv_i64 t1 = tcg_temp_new_i64();
2416 TCGv_i64 t2 = tcg_temp_new_i64();
2417 TCGv_i64 t3 = tcg_temp_new_i64();
2418 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2419 /* Adjust for negative inputs. */
2420 tcg_gen_sari_i64(t2, arg1, 63);
2421 tcg_gen_sari_i64(t3, arg2, 63);
2422 tcg_gen_and_i64(t2, t2, arg2);
2423 tcg_gen_and_i64(t3, t3, arg1);
2424 tcg_gen_sub_i64(rh, t1, t2);
2425 tcg_gen_sub_i64(rh, rh, t3);
2426 tcg_gen_mov_i64(rl, t0);
2427 tcg_temp_free_i64(t0);
2428 tcg_temp_free_i64(t1);
2429 tcg_temp_free_i64(t2);
2430 tcg_temp_free_i64(t3);
2431 } else {
2432 TCGv_i64 t0 = tcg_temp_new_i64();
2433 tcg_gen_mul_i64(t0, arg1, arg2);
2434 gen_helper_mulsh_i64(rh, arg1, arg2);
2435 tcg_gen_mov_i64(rl, t0);
2436 tcg_temp_free_i64(t0);
2437 }
2438 }
2439
2440 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2441 {
2442 TCGv_i64 t0 = tcg_temp_new_i64();
2443 TCGv_i64 t1 = tcg_temp_new_i64();
2444 TCGv_i64 t2 = tcg_temp_new_i64();
2445 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2446 /* Adjust for negative input for the signed arg1. */
2447 tcg_gen_sari_i64(t2, arg1, 63);
2448 tcg_gen_and_i64(t2, t2, arg2);
2449 tcg_gen_sub_i64(rh, t1, t2);
2450 tcg_gen_mov_i64(rl, t0);
2451 tcg_temp_free_i64(t0);
2452 tcg_temp_free_i64(t1);
2453 tcg_temp_free_i64(t2);
2454 }
2455
2456 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2457 {
2458 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2459 }
2460
2461 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2462 {
2463 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2464 }
2465
2466 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2467 {
2468 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2469 }
2470
2471 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2472 {
2473 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2474 }
2475
2476 /* Size changing operations. */
2477
2478 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2479 {
2480 if (TCG_TARGET_REG_BITS == 32) {
2481 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2482 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2483 tcg_gen_op2(INDEX_op_extrl_i64_i32,
2484 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2485 } else {
2486 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2487 }
2488 }
2489
2490 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2491 {
2492 if (TCG_TARGET_REG_BITS == 32) {
2493 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2494 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2495 tcg_gen_op2(INDEX_op_extrh_i64_i32,
2496 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2497 } else {
2498 TCGv_i64 t = tcg_temp_new_i64();
2499 tcg_gen_shri_i64(t, arg, 32);
2500 tcg_gen_mov_i32(ret, (TCGv_i32)t);
2501 tcg_temp_free_i64(t);
2502 }
2503 }
2504
2505 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2506 {
2507 if (TCG_TARGET_REG_BITS == 32) {
2508 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2509 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2510 } else {
2511 tcg_gen_op2(INDEX_op_extu_i32_i64,
2512 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2513 }
2514 }
2515
2516 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2517 {
2518 if (TCG_TARGET_REG_BITS == 32) {
2519 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2520 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2521 } else {
2522 tcg_gen_op2(INDEX_op_ext_i32_i64,
2523 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2524 }
2525 }
2526
2527 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2528 {
2529 TCGv_i64 tmp;
2530
2531 if (TCG_TARGET_REG_BITS == 32) {
2532 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2533 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2534 return;
2535 }
2536
2537 tmp = tcg_temp_new_i64();
2538 /* These extensions are only needed for type correctness.
2539 We may be able to do better given target specific information. */
2540 tcg_gen_extu_i32_i64(tmp, high);
2541 tcg_gen_extu_i32_i64(dest, low);
2542 /* If deposit is available, use it. Otherwise use the extra
2543 knowledge that we have of the zero-extensions above. */
2544 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2545 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2546 } else {
2547 tcg_gen_shli_i64(tmp, tmp, 32);
2548 tcg_gen_or_i64(dest, dest, tmp);
2549 }
2550 tcg_temp_free_i64(tmp);
2551 }
2552
2553 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2554 {
2555 if (TCG_TARGET_REG_BITS == 32) {
2556 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2557 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2558 } else {
2559 tcg_gen_extrl_i64_i32(lo, arg);
2560 tcg_gen_extrh_i64_i32(hi, arg);
2561 }
2562 }
2563
2564 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2565 {
2566 tcg_gen_ext32u_i64(lo, arg);
2567 tcg_gen_shri_i64(hi, arg, 32);
2568 }
2569
2570 /* QEMU specific operations. */
2571
2572 void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
2573 {
2574 uintptr_t val = (uintptr_t)tb + idx;
2575
2576 if (tb == NULL) {
2577 tcg_debug_assert(idx == 0);
2578 } else if (idx <= TB_EXIT_IDXMAX) {
2579 #ifdef CONFIG_DEBUG_TCG
2580 /* This is an exit following a goto_tb. Verify that we have
2581 seen this numbered exit before, via tcg_gen_goto_tb. */
2582 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2583 #endif
2584 /* When not chaining, exit without indicating a link. */
2585 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2586 val = 0;
2587 }
2588 } else {
2589 /* This is an exit via the exitreq label. */
2590 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2591 }
2592
2593 tcg_gen_op1i(INDEX_op_exit_tb, val);
2594 }
2595
2596 void tcg_gen_goto_tb(unsigned idx)
2597 {
2598 /* We only support two chained exits. */
2599 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2600 #ifdef CONFIG_DEBUG_TCG
2601 /* Verify that we havn't seen this numbered exit before. */
2602 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2603 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2604 #endif
2605 /* When not chaining, we simply fall through to the "fallback" exit. */
2606 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2607 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2608 }
2609 }
2610
2611 void tcg_gen_lookup_and_goto_ptr(void)
2612 {
2613 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2614 TCGv_ptr ptr = tcg_temp_new_ptr();
2615 gen_helper_lookup_tb_ptr(ptr, cpu_env);
2616 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2617 tcg_temp_free_ptr(ptr);
2618 } else {
2619 tcg_gen_exit_tb(NULL, 0);
2620 }
2621 }
2622
2623 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2624 {
2625 /* Trigger the asserts within as early as possible. */
2626 (void)get_alignment_bits(op);
2627
2628 switch (op & MO_SIZE) {
2629 case MO_8:
2630 op &= ~MO_BSWAP;
2631 break;
2632 case MO_16:
2633 break;
2634 case MO_32:
2635 if (!is64) {
2636 op &= ~MO_SIGN;
2637 }
2638 break;
2639 case MO_64:
2640 if (!is64) {
2641 tcg_abort();
2642 }
2643 break;
2644 }
2645 if (st) {
2646 op &= ~MO_SIGN;
2647 }
2648 return op;
2649 }
2650
2651 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2652 TCGMemOp memop, TCGArg idx)
2653 {
2654 TCGMemOpIdx oi = make_memop_idx(memop, idx);
2655 #if TARGET_LONG_BITS == 32
2656 tcg_gen_op3i_i32(opc, val, addr, oi);
2657 #else
2658 if (TCG_TARGET_REG_BITS == 32) {
2659 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2660 } else {
2661 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2662 }
2663 #endif
2664 }
2665
2666 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2667 TCGMemOp memop, TCGArg idx)
2668 {
2669 TCGMemOpIdx oi = make_memop_idx(memop, idx);
2670 #if TARGET_LONG_BITS == 32
2671 if (TCG_TARGET_REG_BITS == 32) {
2672 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2673 } else {
2674 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2675 }
2676 #else
2677 if (TCG_TARGET_REG_BITS == 32) {
2678 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2679 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2680 } else {
2681 tcg_gen_op3i_i64(opc, val, addr, oi);
2682 }
2683 #endif
2684 }
2685
2686 static void tcg_gen_req_mo(TCGBar type)
2687 {
2688 #ifdef TCG_GUEST_DEFAULT_MO
2689 type &= TCG_GUEST_DEFAULT_MO;
2690 #endif
2691 type &= ~TCG_TARGET_DEFAULT_MO;
2692 if (type) {
2693 tcg_gen_mb(type | TCG_BAR_SC);
2694 }
2695 }
2696
2697 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2698 {
2699 TCGMemOp orig_memop;
2700
2701 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2702 memop = tcg_canonicalize_memop(memop, 0, 0);
2703 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2704 addr, trace_mem_get_info(memop, 0));
2705
2706 orig_memop = memop;
2707 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2708 memop &= ~MO_BSWAP;
2709 /* The bswap primitive requires zero-extended input. */
2710 if ((memop & MO_SSIZE) == MO_SW) {
2711 memop &= ~MO_SIGN;
2712 }
2713 }
2714
2715 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2716
2717 if ((orig_memop ^ memop) & MO_BSWAP) {
2718 switch (orig_memop & MO_SIZE) {
2719 case MO_16:
2720 tcg_gen_bswap16_i32(val, val);
2721 if (orig_memop & MO_SIGN) {
2722 tcg_gen_ext16s_i32(val, val);
2723 }
2724 break;
2725 case MO_32:
2726 tcg_gen_bswap32_i32(val, val);
2727 break;
2728 default:
2729 g_assert_not_reached();
2730 }
2731 }
2732 }
2733
2734 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2735 {
2736 TCGv_i32 swap = NULL;
2737
2738 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2739 memop = tcg_canonicalize_memop(memop, 0, 1);
2740 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2741 addr, trace_mem_get_info(memop, 1));
2742
2743 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2744 swap = tcg_temp_new_i32();
2745 switch (memop & MO_SIZE) {
2746 case MO_16:
2747 tcg_gen_ext16u_i32(swap, val);
2748 tcg_gen_bswap16_i32(swap, swap);
2749 break;
2750 case MO_32:
2751 tcg_gen_bswap32_i32(swap, val);
2752 break;
2753 default:
2754 g_assert_not_reached();
2755 }
2756 val = swap;
2757 memop &= ~MO_BSWAP;
2758 }
2759
2760 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2761
2762 if (swap) {
2763 tcg_temp_free_i32(swap);
2764 }
2765 }
2766
2767 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2768 {
2769 TCGMemOp orig_memop;
2770
2771 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2772 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2773 if (memop & MO_SIGN) {
2774 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2775 } else {
2776 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2777 }
2778 return;
2779 }
2780
2781 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2782 memop = tcg_canonicalize_memop(memop, 1, 0);
2783 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2784 addr, trace_mem_get_info(memop, 0));
2785
2786 orig_memop = memop;
2787 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2788 memop &= ~MO_BSWAP;
2789 /* The bswap primitive requires zero-extended input. */
2790 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
2791 memop &= ~MO_SIGN;
2792 }
2793 }
2794
2795 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2796
2797 if ((orig_memop ^ memop) & MO_BSWAP) {
2798 switch (orig_memop & MO_SIZE) {
2799 case MO_16:
2800 tcg_gen_bswap16_i64(val, val);
2801 if (orig_memop & MO_SIGN) {
2802 tcg_gen_ext16s_i64(val, val);
2803 }
2804 break;
2805 case MO_32:
2806 tcg_gen_bswap32_i64(val, val);
2807 if (orig_memop & MO_SIGN) {
2808 tcg_gen_ext32s_i64(val, val);
2809 }
2810 break;
2811 case MO_64:
2812 tcg_gen_bswap64_i64(val, val);
2813 break;
2814 default:
2815 g_assert_not_reached();
2816 }
2817 }
2818 }
2819
2820 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2821 {
2822 TCGv_i64 swap = NULL;
2823
2824 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2825 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2826 return;
2827 }
2828
2829 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2830 memop = tcg_canonicalize_memop(memop, 1, 1);
2831 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2832 addr, trace_mem_get_info(memop, 1));
2833
2834 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2835 swap = tcg_temp_new_i64();
2836 switch (memop & MO_SIZE) {
2837 case MO_16:
2838 tcg_gen_ext16u_i64(swap, val);
2839 tcg_gen_bswap16_i64(swap, swap);
2840 break;
2841 case MO_32:
2842 tcg_gen_ext32u_i64(swap, val);
2843 tcg_gen_bswap32_i64(swap, swap);
2844 break;
2845 case MO_64:
2846 tcg_gen_bswap64_i64(swap, val);
2847 break;
2848 default:
2849 g_assert_not_reached();
2850 }
2851 val = swap;
2852 memop &= ~MO_BSWAP;
2853 }
2854
2855 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
2856
2857 if (swap) {
2858 tcg_temp_free_i64(swap);
2859 }
2860 }
2861
2862 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2863 {
2864 switch (opc & MO_SSIZE) {
2865 case MO_SB:
2866 tcg_gen_ext8s_i32(ret, val);
2867 break;
2868 case MO_UB:
2869 tcg_gen_ext8u_i32(ret, val);
2870 break;
2871 case MO_SW:
2872 tcg_gen_ext16s_i32(ret, val);
2873 break;
2874 case MO_UW:
2875 tcg_gen_ext16u_i32(ret, val);
2876 break;
2877 default:
2878 tcg_gen_mov_i32(ret, val);
2879 break;
2880 }
2881 }
2882
2883 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2884 {
2885 switch (opc & MO_SSIZE) {
2886 case MO_SB:
2887 tcg_gen_ext8s_i64(ret, val);
2888 break;
2889 case MO_UB:
2890 tcg_gen_ext8u_i64(ret, val);
2891 break;
2892 case MO_SW:
2893 tcg_gen_ext16s_i64(ret, val);
2894 break;
2895 case MO_UW:
2896 tcg_gen_ext16u_i64(ret, val);
2897 break;
2898 case MO_SL:
2899 tcg_gen_ext32s_i64(ret, val);
2900 break;
2901 case MO_UL:
2902 tcg_gen_ext32u_i64(ret, val);
2903 break;
2904 default:
2905 tcg_gen_mov_i64(ret, val);
2906 break;
2907 }
2908 }
2909
2910 #ifdef CONFIG_SOFTMMU
2911 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2912 TCGv_i32, TCGv_i32, TCGv_i32);
2913 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2914 TCGv_i64, TCGv_i64, TCGv_i32);
2915 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2916 TCGv_i32, TCGv_i32);
2917 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2918 TCGv_i64, TCGv_i32);
2919 #else
2920 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2921 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2922 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2923 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2924 #endif
2925
2926 #ifdef CONFIG_ATOMIC64
2927 # define WITH_ATOMIC64(X) X,
2928 #else
2929 # define WITH_ATOMIC64(X)
2930 #endif
2931
2932 static void * const table_cmpxchg[16] = {
2933 [MO_8] = gen_helper_atomic_cmpxchgb,
2934 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2935 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2936 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2937 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
2938 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2939 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
2940 };
2941
2942 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2943 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2944 {
2945 memop = tcg_canonicalize_memop(memop, 0, 0);
2946
2947 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
2948 TCGv_i32 t1 = tcg_temp_new_i32();
2949 TCGv_i32 t2 = tcg_temp_new_i32();
2950
2951 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2952
2953 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2954 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2955 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2956 tcg_temp_free_i32(t2);
2957
2958 if (memop & MO_SIGN) {
2959 tcg_gen_ext_i32(retv, t1, memop);
2960 } else {
2961 tcg_gen_mov_i32(retv, t1);
2962 }
2963 tcg_temp_free_i32(t1);
2964 } else {
2965 gen_atomic_cx_i32 gen;
2966
2967 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2968 tcg_debug_assert(gen != NULL);
2969
2970 #ifdef CONFIG_SOFTMMU
2971 {
2972 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2973 gen(retv, cpu_env, addr, cmpv, newv, oi);
2974 tcg_temp_free_i32(oi);
2975 }
2976 #else
2977 gen(retv, cpu_env, addr, cmpv, newv);
2978 #endif
2979
2980 if (memop & MO_SIGN) {
2981 tcg_gen_ext_i32(retv, retv, memop);
2982 }
2983 }
2984 }
2985
2986 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2987 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2988 {
2989 memop = tcg_canonicalize_memop(memop, 1, 0);
2990
2991 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
2992 TCGv_i64 t1 = tcg_temp_new_i64();
2993 TCGv_i64 t2 = tcg_temp_new_i64();
2994
2995 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2996
2997 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2998 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2999 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3000 tcg_temp_free_i64(t2);
3001
3002 if (memop & MO_SIGN) {
3003 tcg_gen_ext_i64(retv, t1, memop);
3004 } else {
3005 tcg_gen_mov_i64(retv, t1);
3006 }
3007 tcg_temp_free_i64(t1);
3008 } else if ((memop & MO_SIZE) == MO_64) {
3009 #ifdef CONFIG_ATOMIC64
3010 gen_atomic_cx_i64 gen;
3011
3012 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3013 tcg_debug_assert(gen != NULL);
3014
3015 #ifdef CONFIG_SOFTMMU
3016 {
3017 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
3018 gen(retv, cpu_env, addr, cmpv, newv, oi);
3019 tcg_temp_free_i32(oi);
3020 }
3021 #else
3022 gen(retv, cpu_env, addr, cmpv, newv);
3023 #endif
3024 #else
3025 gen_helper_exit_atomic(cpu_env);
3026 /* Produce a result, so that we have a well-formed opcode stream
3027 with respect to uses of the result in the (dead) code following. */
3028 tcg_gen_movi_i64(retv, 0);
3029 #endif /* CONFIG_ATOMIC64 */
3030 } else {
3031 TCGv_i32 c32 = tcg_temp_new_i32();
3032 TCGv_i32 n32 = tcg_temp_new_i32();
3033 TCGv_i32 r32 = tcg_temp_new_i32();
3034
3035 tcg_gen_extrl_i64_i32(c32, cmpv);
3036 tcg_gen_extrl_i64_i32(n32, newv);
3037 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3038 tcg_temp_free_i32(c32);
3039 tcg_temp_free_i32(n32);
3040
3041 tcg_gen_extu_i32_i64(retv, r32);
3042 tcg_temp_free_i32(r32);
3043
3044 if (memop & MO_SIGN) {
3045 tcg_gen_ext_i64(retv, retv, memop);
3046 }
3047 }
3048 }
3049
3050 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3051 TCGArg idx, TCGMemOp memop, bool new_val,
3052 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3053 {
3054 TCGv_i32 t1 = tcg_temp_new_i32();
3055 TCGv_i32 t2 = tcg_temp_new_i32();
3056
3057 memop = tcg_canonicalize_memop(memop, 0, 0);
3058
3059 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3060 gen(t2, t1, val);
3061 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3062
3063 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3064 tcg_temp_free_i32(t1);
3065 tcg_temp_free_i32(t2);
3066 }
3067
3068 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3069 TCGArg idx, TCGMemOp memop, void * const table[])
3070 {
3071 gen_atomic_op_i32 gen;
3072
3073 memop = tcg_canonicalize_memop(memop, 0, 0);
3074
3075 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3076 tcg_debug_assert(gen != NULL);
3077
3078 #ifdef CONFIG_SOFTMMU
3079 {
3080 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3081 gen(ret, cpu_env, addr, val, oi);
3082 tcg_temp_free_i32(oi);
3083 }
3084 #else
3085 gen(ret, cpu_env, addr, val);
3086 #endif
3087
3088 if (memop & MO_SIGN) {
3089 tcg_gen_ext_i32(ret, ret, memop);
3090 }
3091 }
3092
3093 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3094 TCGArg idx, TCGMemOp memop, bool new_val,
3095 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3096 {
3097 TCGv_i64 t1 = tcg_temp_new_i64();
3098 TCGv_i64 t2 = tcg_temp_new_i64();
3099
3100 memop = tcg_canonicalize_memop(memop, 1, 0);
3101
3102 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3103 gen(t2, t1, val);
3104 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3105
3106 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3107 tcg_temp_free_i64(t1);
3108 tcg_temp_free_i64(t2);
3109 }
3110
3111 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3112 TCGArg idx, TCGMemOp memop, void * const table[])
3113 {
3114 memop = tcg_canonicalize_memop(memop, 1, 0);
3115
3116 if ((memop & MO_SIZE) == MO_64) {
3117 #ifdef CONFIG_ATOMIC64
3118 gen_atomic_op_i64 gen;
3119
3120 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3121 tcg_debug_assert(gen != NULL);
3122
3123 #ifdef CONFIG_SOFTMMU
3124 {
3125 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3126 gen(ret, cpu_env, addr, val, oi);
3127 tcg_temp_free_i32(oi);
3128 }
3129 #else
3130 gen(ret, cpu_env, addr, val);
3131 #endif
3132 #else
3133 gen_helper_exit_atomic(cpu_env);
3134 /* Produce a result, so that we have a well-formed opcode stream
3135 with respect to uses of the result in the (dead) code following. */
3136 tcg_gen_movi_i64(ret, 0);
3137 #endif /* CONFIG_ATOMIC64 */
3138 } else {
3139 TCGv_i32 v32 = tcg_temp_new_i32();
3140 TCGv_i32 r32 = tcg_temp_new_i32();
3141
3142 tcg_gen_extrl_i64_i32(v32, val);
3143 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3144 tcg_temp_free_i32(v32);
3145
3146 tcg_gen_extu_i32_i64(ret, r32);
3147 tcg_temp_free_i32(r32);
3148
3149 if (memop & MO_SIGN) {
3150 tcg_gen_ext_i64(ret, ret, memop);
3151 }
3152 }
3153 }
3154
3155 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3156 static void * const table_##NAME[16] = { \
3157 [MO_8] = gen_helper_atomic_##NAME##b, \
3158 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3159 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3160 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3161 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
3162 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3163 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
3164 }; \
3165 void tcg_gen_atomic_##NAME##_i32 \
3166 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
3167 { \
3168 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3169 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3170 } else { \
3171 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3172 tcg_gen_##OP##_i32); \
3173 } \
3174 } \
3175 void tcg_gen_atomic_##NAME##_i64 \
3176 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
3177 { \
3178 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3179 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3180 } else { \
3181 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3182 tcg_gen_##OP##_i64); \
3183 } \
3184 }
3185
3186 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3187 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3188 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3189 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3190 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3191 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3192 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3193 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3194
3195 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3196 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3197 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3198 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3199 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3200 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3201 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3202 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3203
3204 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3205 {
3206 tcg_gen_mov_i32(r, b);
3207 }
3208
3209 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3210 {
3211 tcg_gen_mov_i64(r, b);
3212 }
3213
3214 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3215
3216 #undef GEN_ATOMIC_HELPER
QEmu