?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
target/i386: reimplement 0x0f 0x38, add AVX
[qemu.git] / tcg / tcg-op.c
blob019fab00ccb6215083b487d463248d3989869543
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "exec/exec-all.h"
27 #include "tcg/tcg.h"
28 #include "tcg/tcg-op.h"
29 #include "tcg/tcg-mo.h"
30 #include "exec/plugin-gen.h"
31
32 /* Reduce the number of ifdefs below. This assumes that all uses of
33 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
34 the compiler can eliminate. */
35 #if TCG_TARGET_REG_BITS == 64
36 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
37 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
38 #define TCGV_LOW TCGV_LOW_link_error
39 #define TCGV_HIGH TCGV_HIGH_link_error
40 #endif
41
42 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
43 {
44 TCGOp *op = tcg_emit_op(opc);
45 op->args[0] = a1;
46 }
47
48 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
49 {
50 TCGOp *op = tcg_emit_op(opc);
51 op->args[0] = a1;
52 op->args[1] = a2;
53 }
54
55 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
56 {
57 TCGOp *op = tcg_emit_op(opc);
58 op->args[0] = a1;
59 op->args[1] = a2;
60 op->args[2] = a3;
61 }
62
63 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
64 {
65 TCGOp *op = tcg_emit_op(opc);
66 op->args[0] = a1;
67 op->args[1] = a2;
68 op->args[2] = a3;
69 op->args[3] = a4;
70 }
71
72 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
73 TCGArg a4, TCGArg a5)
74 {
75 TCGOp *op = tcg_emit_op(opc);
76 op->args[0] = a1;
77 op->args[1] = a2;
78 op->args[2] = a3;
79 op->args[3] = a4;
80 op->args[4] = a5;
81 }
82
83 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
84 TCGArg a4, TCGArg a5, TCGArg a6)
85 {
86 TCGOp *op = tcg_emit_op(opc);
87 op->args[0] = a1;
88 op->args[1] = a2;
89 op->args[2] = a3;
90 op->args[3] = a4;
91 op->args[4] = a5;
92 op->args[5] = a6;
93 }
94
95 void tcg_gen_mb(TCGBar mb_type)
96 {
97 if (tcg_ctx->tb_cflags & CF_PARALLEL) {
98 tcg_gen_op1(INDEX_op_mb, mb_type);
99 }
100 }
101
102 /* 32 bit ops */
103
104 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
105 {
106 tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
107 }
108
109 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
110 {
111 /* some cases can be optimized here */
112 if (arg2 == 0) {
113 tcg_gen_mov_i32(ret, arg1);
114 } else {
115 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
116 }
117 }
118
119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
120 {
121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
122 /* Don't recurse with tcg_gen_neg_i32. */
123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
124 } else {
125 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
126 }
127 }
128
129 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
130 {
131 /* some cases can be optimized here */
132 if (arg2 == 0) {
133 tcg_gen_mov_i32(ret, arg1);
134 } else {
135 tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2));
136 }
137 }
138
139 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
140 {
141 /* Some cases can be optimized here. */
142 switch (arg2) {
143 case 0:
144 tcg_gen_movi_i32(ret, 0);
145 return;
146 case -1:
147 tcg_gen_mov_i32(ret, arg1);
148 return;
149 case 0xff:
150 /* Don't recurse with tcg_gen_ext8u_i32. */
151 if (TCG_TARGET_HAS_ext8u_i32) {
152 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
153 return;
154 }
155 break;
156 case 0xffff:
157 if (TCG_TARGET_HAS_ext16u_i32) {
158 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
159 return;
160 }
161 break;
162 }
163
164 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
165 }
166
167 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
168 {
169 /* Some cases can be optimized here. */
170 if (arg2 == -1) {
171 tcg_gen_movi_i32(ret, -1);
172 } else if (arg2 == 0) {
173 tcg_gen_mov_i32(ret, arg1);
174 } else {
175 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
176 }
177 }
178
179 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
180 {
181 /* Some cases can be optimized here. */
182 if (arg2 == 0) {
183 tcg_gen_mov_i32(ret, arg1);
184 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
185 /* Don't recurse with tcg_gen_not_i32. */
186 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
187 } else {
188 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
189 }
190 }
191
192 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
193 {
194 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
195 if (arg2 == 0) {
196 tcg_gen_mov_i32(ret, arg1);
197 } else {
198 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
199 }
200 }
201
202 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
203 {
204 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
205 if (arg2 == 0) {
206 tcg_gen_mov_i32(ret, arg1);
207 } else {
208 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
209 }
210 }
211
212 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
213 {
214 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
215 if (arg2 == 0) {
216 tcg_gen_mov_i32(ret, arg1);
217 } else {
218 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
219 }
220 }
221
222 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
223 {
224 if (cond == TCG_COND_ALWAYS) {
225 tcg_gen_br(l);
226 } else if (cond != TCG_COND_NEVER) {
227 l->refs++;
228 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
229 }
230 }
231
232 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
233 {
234 if (cond == TCG_COND_ALWAYS) {
235 tcg_gen_br(l);
236 } else if (cond != TCG_COND_NEVER) {
237 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
238 }
239 }
240
241 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
242 TCGv_i32 arg1, TCGv_i32 arg2)
243 {
244 if (cond == TCG_COND_ALWAYS) {
245 tcg_gen_movi_i32(ret, 1);
246 } else if (cond == TCG_COND_NEVER) {
247 tcg_gen_movi_i32(ret, 0);
248 } else {
249 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
250 }
251 }
252
253 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
254 TCGv_i32 arg1, int32_t arg2)
255 {
256 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
257 }
258
259 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
260 {
261 if (arg2 == 0) {
262 tcg_gen_movi_i32(ret, 0);
263 } else if (is_power_of_2(arg2)) {
264 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
265 } else {
266 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
267 }
268 }
269
270 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
271 {
272 if (TCG_TARGET_HAS_div_i32) {
273 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
274 } else if (TCG_TARGET_HAS_div2_i32) {
275 TCGv_i32 t0 = tcg_temp_new_i32();
276 tcg_gen_sari_i32(t0, arg1, 31);
277 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
278 tcg_temp_free_i32(t0);
279 } else {
280 gen_helper_div_i32(ret, arg1, arg2);
281 }
282 }
283
284 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
285 {
286 if (TCG_TARGET_HAS_rem_i32) {
287 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
288 } else if (TCG_TARGET_HAS_div_i32) {
289 TCGv_i32 t0 = tcg_temp_new_i32();
290 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
291 tcg_gen_mul_i32(t0, t0, arg2);
292 tcg_gen_sub_i32(ret, arg1, t0);
293 tcg_temp_free_i32(t0);
294 } else if (TCG_TARGET_HAS_div2_i32) {
295 TCGv_i32 t0 = tcg_temp_new_i32();
296 tcg_gen_sari_i32(t0, arg1, 31);
297 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
298 tcg_temp_free_i32(t0);
299 } else {
300 gen_helper_rem_i32(ret, arg1, arg2);
301 }
302 }
303
304 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
305 {
306 if (TCG_TARGET_HAS_div_i32) {
307 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
308 } else if (TCG_TARGET_HAS_div2_i32) {
309 TCGv_i32 t0 = tcg_temp_new_i32();
310 tcg_gen_movi_i32(t0, 0);
311 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
312 tcg_temp_free_i32(t0);
313 } else {
314 gen_helper_divu_i32(ret, arg1, arg2);
315 }
316 }
317
318 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
319 {
320 if (TCG_TARGET_HAS_rem_i32) {
321 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
322 } else if (TCG_TARGET_HAS_div_i32) {
323 TCGv_i32 t0 = tcg_temp_new_i32();
324 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
325 tcg_gen_mul_i32(t0, t0, arg2);
326 tcg_gen_sub_i32(ret, arg1, t0);
327 tcg_temp_free_i32(t0);
328 } else if (TCG_TARGET_HAS_div2_i32) {
329 TCGv_i32 t0 = tcg_temp_new_i32();
330 tcg_gen_movi_i32(t0, 0);
331 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
332 tcg_temp_free_i32(t0);
333 } else {
334 gen_helper_remu_i32(ret, arg1, arg2);
335 }
336 }
337
338 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
339 {
340 if (TCG_TARGET_HAS_andc_i32) {
341 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
342 } else {
343 TCGv_i32 t0 = tcg_temp_new_i32();
344 tcg_gen_not_i32(t0, arg2);
345 tcg_gen_and_i32(ret, arg1, t0);
346 tcg_temp_free_i32(t0);
347 }
348 }
349
350 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
351 {
352 if (TCG_TARGET_HAS_eqv_i32) {
353 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
354 } else {
355 tcg_gen_xor_i32(ret, arg1, arg2);
356 tcg_gen_not_i32(ret, ret);
357 }
358 }
359
360 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
361 {
362 if (TCG_TARGET_HAS_nand_i32) {
363 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
364 } else {
365 tcg_gen_and_i32(ret, arg1, arg2);
366 tcg_gen_not_i32(ret, ret);
367 }
368 }
369
370 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
371 {
372 if (TCG_TARGET_HAS_nor_i32) {
373 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
374 } else {
375 tcg_gen_or_i32(ret, arg1, arg2);
376 tcg_gen_not_i32(ret, ret);
377 }
378 }
379
380 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
381 {
382 if (TCG_TARGET_HAS_orc_i32) {
383 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
384 } else {
385 TCGv_i32 t0 = tcg_temp_new_i32();
386 tcg_gen_not_i32(t0, arg2);
387 tcg_gen_or_i32(ret, arg1, t0);
388 tcg_temp_free_i32(t0);
389 }
390 }
391
392 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
393 {
394 if (TCG_TARGET_HAS_clz_i32) {
395 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
396 } else if (TCG_TARGET_HAS_clz_i64) {
397 TCGv_i64 t1 = tcg_temp_new_i64();
398 TCGv_i64 t2 = tcg_temp_new_i64();
399 tcg_gen_extu_i32_i64(t1, arg1);
400 tcg_gen_extu_i32_i64(t2, arg2);
401 tcg_gen_addi_i64(t2, t2, 32);
402 tcg_gen_clz_i64(t1, t1, t2);
403 tcg_gen_extrl_i64_i32(ret, t1);
404 tcg_temp_free_i64(t1);
405 tcg_temp_free_i64(t2);
406 tcg_gen_subi_i32(ret, ret, 32);
407 } else {
408 gen_helper_clz_i32(ret, arg1, arg2);
409 }
410 }
411
412 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
413 {
414 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
415 }
416
417 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
418 {
419 if (TCG_TARGET_HAS_ctz_i32) {
420 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
421 } else if (TCG_TARGET_HAS_ctz_i64) {
422 TCGv_i64 t1 = tcg_temp_new_i64();
423 TCGv_i64 t2 = tcg_temp_new_i64();
424 tcg_gen_extu_i32_i64(t1, arg1);
425 tcg_gen_extu_i32_i64(t2, arg2);
426 tcg_gen_ctz_i64(t1, t1, t2);
427 tcg_gen_extrl_i64_i32(ret, t1);
428 tcg_temp_free_i64(t1);
429 tcg_temp_free_i64(t2);
430 } else if (TCG_TARGET_HAS_ctpop_i32
431 || TCG_TARGET_HAS_ctpop_i64
432 || TCG_TARGET_HAS_clz_i32
433 || TCG_TARGET_HAS_clz_i64) {
434 TCGv_i32 z, t = tcg_temp_new_i32();
435
436 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
437 tcg_gen_subi_i32(t, arg1, 1);
438 tcg_gen_andc_i32(t, t, arg1);
439 tcg_gen_ctpop_i32(t, t);
440 } else {
441 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
442 tcg_gen_neg_i32(t, arg1);
443 tcg_gen_and_i32(t, t, arg1);
444 tcg_gen_clzi_i32(t, t, 32);
445 tcg_gen_xori_i32(t, t, 31);
446 }
447 z = tcg_constant_i32(0);
448 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
449 tcg_temp_free_i32(t);
450 } else {
451 gen_helper_ctz_i32(ret, arg1, arg2);
452 }
453 }
454
455 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
456 {
457 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
458 /* This equivalence has the advantage of not requiring a fixup. */
459 TCGv_i32 t = tcg_temp_new_i32();
460 tcg_gen_subi_i32(t, arg1, 1);
461 tcg_gen_andc_i32(t, t, arg1);
462 tcg_gen_ctpop_i32(ret, t);
463 tcg_temp_free_i32(t);
464 } else {
465 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
466 }
467 }
468
469 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
470 {
471 if (TCG_TARGET_HAS_clz_i32) {
472 TCGv_i32 t = tcg_temp_new_i32();
473 tcg_gen_sari_i32(t, arg, 31);
474 tcg_gen_xor_i32(t, t, arg);
475 tcg_gen_clzi_i32(t, t, 32);
476 tcg_gen_subi_i32(ret, t, 1);
477 tcg_temp_free_i32(t);
478 } else {
479 gen_helper_clrsb_i32(ret, arg);
480 }
481 }
482
483 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
484 {
485 if (TCG_TARGET_HAS_ctpop_i32) {
486 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
487 } else if (TCG_TARGET_HAS_ctpop_i64) {
488 TCGv_i64 t = tcg_temp_new_i64();
489 tcg_gen_extu_i32_i64(t, arg1);
490 tcg_gen_ctpop_i64(t, t);
491 tcg_gen_extrl_i64_i32(ret, t);
492 tcg_temp_free_i64(t);
493 } else {
494 gen_helper_ctpop_i32(ret, arg1);
495 }
496 }
497
498 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
499 {
500 if (TCG_TARGET_HAS_rot_i32) {
501 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
502 } else {
503 TCGv_i32 t0, t1;
504
505 t0 = tcg_temp_new_i32();
506 t1 = tcg_temp_new_i32();
507 tcg_gen_shl_i32(t0, arg1, arg2);
508 tcg_gen_subfi_i32(t1, 32, arg2);
509 tcg_gen_shr_i32(t1, arg1, t1);
510 tcg_gen_or_i32(ret, t0, t1);
511 tcg_temp_free_i32(t0);
512 tcg_temp_free_i32(t1);
513 }
514 }
515
516 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
517 {
518 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
519 /* some cases can be optimized here */
520 if (arg2 == 0) {
521 tcg_gen_mov_i32(ret, arg1);
522 } else if (TCG_TARGET_HAS_rot_i32) {
523 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
524 } else {
525 TCGv_i32 t0, t1;
526 t0 = tcg_temp_new_i32();
527 t1 = tcg_temp_new_i32();
528 tcg_gen_shli_i32(t0, arg1, arg2);
529 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
530 tcg_gen_or_i32(ret, t0, t1);
531 tcg_temp_free_i32(t0);
532 tcg_temp_free_i32(t1);
533 }
534 }
535
536 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
537 {
538 if (TCG_TARGET_HAS_rot_i32) {
539 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
540 } else {
541 TCGv_i32 t0, t1;
542
543 t0 = tcg_temp_new_i32();
544 t1 = tcg_temp_new_i32();
545 tcg_gen_shr_i32(t0, arg1, arg2);
546 tcg_gen_subfi_i32(t1, 32, arg2);
547 tcg_gen_shl_i32(t1, arg1, t1);
548 tcg_gen_or_i32(ret, t0, t1);
549 tcg_temp_free_i32(t0);
550 tcg_temp_free_i32(t1);
551 }
552 }
553
554 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
555 {
556 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
557 /* some cases can be optimized here */
558 if (arg2 == 0) {
559 tcg_gen_mov_i32(ret, arg1);
560 } else {
561 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
562 }
563 }
564
565 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
566 unsigned int ofs, unsigned int len)
567 {
568 uint32_t mask;
569 TCGv_i32 t1;
570
571 tcg_debug_assert(ofs < 32);
572 tcg_debug_assert(len > 0);
573 tcg_debug_assert(len <= 32);
574 tcg_debug_assert(ofs + len <= 32);
575
576 if (len == 32) {
577 tcg_gen_mov_i32(ret, arg2);
578 return;
579 }
580 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
581 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
582 return;
583 }
584
585 t1 = tcg_temp_new_i32();
586
587 if (TCG_TARGET_HAS_extract2_i32) {
588 if (ofs + len == 32) {
589 tcg_gen_shli_i32(t1, arg1, len);
590 tcg_gen_extract2_i32(ret, t1, arg2, len);
591 goto done;
592 }
593 if (ofs == 0) {
594 tcg_gen_extract2_i32(ret, arg1, arg2, len);
595 tcg_gen_rotli_i32(ret, ret, len);
596 goto done;
597 }
598 }
599
600 mask = (1u << len) - 1;
601 if (ofs + len < 32) {
602 tcg_gen_andi_i32(t1, arg2, mask);
603 tcg_gen_shli_i32(t1, t1, ofs);
604 } else {
605 tcg_gen_shli_i32(t1, arg2, ofs);
606 }
607 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
608 tcg_gen_or_i32(ret, ret, t1);
609 done:
610 tcg_temp_free_i32(t1);
611 }
612
613 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
614 unsigned int ofs, unsigned int len)
615 {
616 tcg_debug_assert(ofs < 32);
617 tcg_debug_assert(len > 0);
618 tcg_debug_assert(len <= 32);
619 tcg_debug_assert(ofs + len <= 32);
620
621 if (ofs + len == 32) {
622 tcg_gen_shli_i32(ret, arg, ofs);
623 } else if (ofs == 0) {
624 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
625 } else if (TCG_TARGET_HAS_deposit_i32
626 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
627 TCGv_i32 zero = tcg_constant_i32(0);
628 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
629 } else {
630 /* To help two-operand hosts we prefer to zero-extend first,
631 which allows ARG to stay live. */
632 switch (len) {
633 case 16:
634 if (TCG_TARGET_HAS_ext16u_i32) {
635 tcg_gen_ext16u_i32(ret, arg);
636 tcg_gen_shli_i32(ret, ret, ofs);
637 return;
638 }
639 break;
640 case 8:
641 if (TCG_TARGET_HAS_ext8u_i32) {
642 tcg_gen_ext8u_i32(ret, arg);
643 tcg_gen_shli_i32(ret, ret, ofs);
644 return;
645 }
646 break;
647 }
648 /* Otherwise prefer zero-extension over AND for code size. */
649 switch (ofs + len) {
650 case 16:
651 if (TCG_TARGET_HAS_ext16u_i32) {
652 tcg_gen_shli_i32(ret, arg, ofs);
653 tcg_gen_ext16u_i32(ret, ret);
654 return;
655 }
656 break;
657 case 8:
658 if (TCG_TARGET_HAS_ext8u_i32) {
659 tcg_gen_shli_i32(ret, arg, ofs);
660 tcg_gen_ext8u_i32(ret, ret);
661 return;
662 }
663 break;
664 }
665 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
666 tcg_gen_shli_i32(ret, ret, ofs);
667 }
668 }
669
670 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
671 unsigned int ofs, unsigned int len)
672 {
673 tcg_debug_assert(ofs < 32);
674 tcg_debug_assert(len > 0);
675 tcg_debug_assert(len <= 32);
676 tcg_debug_assert(ofs + len <= 32);
677
678 /* Canonicalize certain special cases, even if extract is supported. */
679 if (ofs + len == 32) {
680 tcg_gen_shri_i32(ret, arg, 32 - len);
681 return;
682 }
683 if (ofs == 0) {
684 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
685 return;
686 }
687
688 if (TCG_TARGET_HAS_extract_i32
689 && TCG_TARGET_extract_i32_valid(ofs, len)) {
690 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
691 return;
692 }
693
694 /* Assume that zero-extension, if available, is cheaper than a shift. */
695 switch (ofs + len) {
696 case 16:
697 if (TCG_TARGET_HAS_ext16u_i32) {
698 tcg_gen_ext16u_i32(ret, arg);
699 tcg_gen_shri_i32(ret, ret, ofs);
700 return;
701 }
702 break;
703 case 8:
704 if (TCG_TARGET_HAS_ext8u_i32) {
705 tcg_gen_ext8u_i32(ret, arg);
706 tcg_gen_shri_i32(ret, ret, ofs);
707 return;
708 }
709 break;
710 }
711
712 /* ??? Ideally we'd know what values are available for immediate AND.
713 Assume that 8 bits are available, plus the special case of 16,
714 so that we get ext8u, ext16u. */
715 switch (len) {
716 case 1 ... 8: case 16:
717 tcg_gen_shri_i32(ret, arg, ofs);
718 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
719 break;
720 default:
721 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
722 tcg_gen_shri_i32(ret, ret, 32 - len);
723 break;
724 }
725 }
726
727 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
728 unsigned int ofs, unsigned int len)
729 {
730 tcg_debug_assert(ofs < 32);
731 tcg_debug_assert(len > 0);
732 tcg_debug_assert(len <= 32);
733 tcg_debug_assert(ofs + len <= 32);
734
735 /* Canonicalize certain special cases, even if extract is supported. */
736 if (ofs + len == 32) {
737 tcg_gen_sari_i32(ret, arg, 32 - len);
738 return;
739 }
740 if (ofs == 0) {
741 switch (len) {
742 case 16:
743 tcg_gen_ext16s_i32(ret, arg);
744 return;
745 case 8:
746 tcg_gen_ext8s_i32(ret, arg);
747 return;
748 }
749 }
750
751 if (TCG_TARGET_HAS_sextract_i32
752 && TCG_TARGET_extract_i32_valid(ofs, len)) {
753 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
754 return;
755 }
756
757 /* Assume that sign-extension, if available, is cheaper than a shift. */
758 switch (ofs + len) {
759 case 16:
760 if (TCG_TARGET_HAS_ext16s_i32) {
761 tcg_gen_ext16s_i32(ret, arg);
762 tcg_gen_sari_i32(ret, ret, ofs);
763 return;
764 }
765 break;
766 case 8:
767 if (TCG_TARGET_HAS_ext8s_i32) {
768 tcg_gen_ext8s_i32(ret, arg);
769 tcg_gen_sari_i32(ret, ret, ofs);
770 return;
771 }
772 break;
773 }
774 switch (len) {
775 case 16:
776 if (TCG_TARGET_HAS_ext16s_i32) {
777 tcg_gen_shri_i32(ret, arg, ofs);
778 tcg_gen_ext16s_i32(ret, ret);
779 return;
780 }
781 break;
782 case 8:
783 if (TCG_TARGET_HAS_ext8s_i32) {
784 tcg_gen_shri_i32(ret, arg, ofs);
785 tcg_gen_ext8s_i32(ret, ret);
786 return;
787 }
788 break;
789 }
790
791 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
792 tcg_gen_sari_i32(ret, ret, 32 - len);
793 }
794
795 /*
796 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
797 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
798 */
799 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
800 unsigned int ofs)
801 {
802 tcg_debug_assert(ofs <= 32);
803 if (ofs == 0) {
804 tcg_gen_mov_i32(ret, al);
805 } else if (ofs == 32) {
806 tcg_gen_mov_i32(ret, ah);
807 } else if (al == ah) {
808 tcg_gen_rotri_i32(ret, al, ofs);
809 } else if (TCG_TARGET_HAS_extract2_i32) {
810 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
811 } else {
812 TCGv_i32 t0 = tcg_temp_new_i32();
813 tcg_gen_shri_i32(t0, al, ofs);
814 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
815 tcg_temp_free_i32(t0);
816 }
817 }
818
819 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
820 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
821 {
822 if (cond == TCG_COND_ALWAYS) {
823 tcg_gen_mov_i32(ret, v1);
824 } else if (cond == TCG_COND_NEVER) {
825 tcg_gen_mov_i32(ret, v2);
826 } else if (TCG_TARGET_HAS_movcond_i32) {
827 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
828 } else {
829 TCGv_i32 t0 = tcg_temp_new_i32();
830 TCGv_i32 t1 = tcg_temp_new_i32();
831 tcg_gen_setcond_i32(cond, t0, c1, c2);
832 tcg_gen_neg_i32(t0, t0);
833 tcg_gen_and_i32(t1, v1, t0);
834 tcg_gen_andc_i32(ret, v2, t0);
835 tcg_gen_or_i32(ret, ret, t1);
836 tcg_temp_free_i32(t0);
837 tcg_temp_free_i32(t1);
838 }
839 }
840
841 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
842 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
843 {
844 if (TCG_TARGET_HAS_add2_i32) {
845 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
846 } else {
847 TCGv_i64 t0 = tcg_temp_new_i64();
848 TCGv_i64 t1 = tcg_temp_new_i64();
849 tcg_gen_concat_i32_i64(t0, al, ah);
850 tcg_gen_concat_i32_i64(t1, bl, bh);
851 tcg_gen_add_i64(t0, t0, t1);
852 tcg_gen_extr_i64_i32(rl, rh, t0);
853 tcg_temp_free_i64(t0);
854 tcg_temp_free_i64(t1);
855 }
856 }
857
858 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
859 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
860 {
861 if (TCG_TARGET_HAS_sub2_i32) {
862 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
863 } else {
864 TCGv_i64 t0 = tcg_temp_new_i64();
865 TCGv_i64 t1 = tcg_temp_new_i64();
866 tcg_gen_concat_i32_i64(t0, al, ah);
867 tcg_gen_concat_i32_i64(t1, bl, bh);
868 tcg_gen_sub_i64(t0, t0, t1);
869 tcg_gen_extr_i64_i32(rl, rh, t0);
870 tcg_temp_free_i64(t0);
871 tcg_temp_free_i64(t1);
872 }
873 }
874
875 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
876 {
877 if (TCG_TARGET_HAS_mulu2_i32) {
878 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
879 } else if (TCG_TARGET_HAS_muluh_i32) {
880 TCGv_i32 t = tcg_temp_new_i32();
881 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
882 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
883 tcg_gen_mov_i32(rl, t);
884 tcg_temp_free_i32(t);
885 } else {
886 TCGv_i64 t0 = tcg_temp_new_i64();
887 TCGv_i64 t1 = tcg_temp_new_i64();
888 tcg_gen_extu_i32_i64(t0, arg1);
889 tcg_gen_extu_i32_i64(t1, arg2);
890 tcg_gen_mul_i64(t0, t0, t1);
891 tcg_gen_extr_i64_i32(rl, rh, t0);
892 tcg_temp_free_i64(t0);
893 tcg_temp_free_i64(t1);
894 }
895 }
896
897 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
898 {
899 if (TCG_TARGET_HAS_muls2_i32) {
900 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
901 } else if (TCG_TARGET_HAS_mulsh_i32) {
902 TCGv_i32 t = tcg_temp_new_i32();
903 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
904 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
905 tcg_gen_mov_i32(rl, t);
906 tcg_temp_free_i32(t);
907 } else if (TCG_TARGET_REG_BITS == 32) {
908 TCGv_i32 t0 = tcg_temp_new_i32();
909 TCGv_i32 t1 = tcg_temp_new_i32();
910 TCGv_i32 t2 = tcg_temp_new_i32();
911 TCGv_i32 t3 = tcg_temp_new_i32();
912 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
913 /* Adjust for negative inputs. */
914 tcg_gen_sari_i32(t2, arg1, 31);
915 tcg_gen_sari_i32(t3, arg2, 31);
916 tcg_gen_and_i32(t2, t2, arg2);
917 tcg_gen_and_i32(t3, t3, arg1);
918 tcg_gen_sub_i32(rh, t1, t2);
919 tcg_gen_sub_i32(rh, rh, t3);
920 tcg_gen_mov_i32(rl, t0);
921 tcg_temp_free_i32(t0);
922 tcg_temp_free_i32(t1);
923 tcg_temp_free_i32(t2);
924 tcg_temp_free_i32(t3);
925 } else {
926 TCGv_i64 t0 = tcg_temp_new_i64();
927 TCGv_i64 t1 = tcg_temp_new_i64();
928 tcg_gen_ext_i32_i64(t0, arg1);
929 tcg_gen_ext_i32_i64(t1, arg2);
930 tcg_gen_mul_i64(t0, t0, t1);
931 tcg_gen_extr_i64_i32(rl, rh, t0);
932 tcg_temp_free_i64(t0);
933 tcg_temp_free_i64(t1);
934 }
935 }
936
937 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
938 {
939 if (TCG_TARGET_REG_BITS == 32) {
940 TCGv_i32 t0 = tcg_temp_new_i32();
941 TCGv_i32 t1 = tcg_temp_new_i32();
942 TCGv_i32 t2 = tcg_temp_new_i32();
943 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
944 /* Adjust for negative input for the signed arg1. */
945 tcg_gen_sari_i32(t2, arg1, 31);
946 tcg_gen_and_i32(t2, t2, arg2);
947 tcg_gen_sub_i32(rh, t1, t2);
948 tcg_gen_mov_i32(rl, t0);
949 tcg_temp_free_i32(t0);
950 tcg_temp_free_i32(t1);
951 tcg_temp_free_i32(t2);
952 } else {
953 TCGv_i64 t0 = tcg_temp_new_i64();
954 TCGv_i64 t1 = tcg_temp_new_i64();
955 tcg_gen_ext_i32_i64(t0, arg1);
956 tcg_gen_extu_i32_i64(t1, arg2);
957 tcg_gen_mul_i64(t0, t0, t1);
958 tcg_gen_extr_i64_i32(rl, rh, t0);
959 tcg_temp_free_i64(t0);
960 tcg_temp_free_i64(t1);
961 }
962 }
963
964 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
965 {
966 if (TCG_TARGET_HAS_ext8s_i32) {
967 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
968 } else {
969 tcg_gen_shli_i32(ret, arg, 24);
970 tcg_gen_sari_i32(ret, ret, 24);
971 }
972 }
973
974 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
975 {
976 if (TCG_TARGET_HAS_ext16s_i32) {
977 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
978 } else {
979 tcg_gen_shli_i32(ret, arg, 16);
980 tcg_gen_sari_i32(ret, ret, 16);
981 }
982 }
983
984 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
985 {
986 if (TCG_TARGET_HAS_ext8u_i32) {
987 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
988 } else {
989 tcg_gen_andi_i32(ret, arg, 0xffu);
990 }
991 }
992
993 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
994 {
995 if (TCG_TARGET_HAS_ext16u_i32) {
996 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
997 } else {
998 tcg_gen_andi_i32(ret, arg, 0xffffu);
999 }
1000 }
1001
1002 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
1003 {
1004 /* Only one extension flag may be present. */
1005 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1006
1007 if (TCG_TARGET_HAS_bswap16_i32) {
1008 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
1009 } else {
1010 TCGv_i32 t0 = tcg_temp_new_i32();
1011 TCGv_i32 t1 = tcg_temp_new_i32();
1012
1013 tcg_gen_shri_i32(t0, arg, 8);
1014 if (!(flags & TCG_BSWAP_IZ)) {
1015 tcg_gen_ext8u_i32(t0, t0);
1016 }
1017
1018 if (flags & TCG_BSWAP_OS) {
1019 tcg_gen_shli_i32(t1, arg, 24);
1020 tcg_gen_sari_i32(t1, t1, 16);
1021 } else if (flags & TCG_BSWAP_OZ) {
1022 tcg_gen_ext8u_i32(t1, arg);
1023 tcg_gen_shli_i32(t1, t1, 8);
1024 } else {
1025 tcg_gen_shli_i32(t1, arg, 8);
1026 }
1027
1028 tcg_gen_or_i32(ret, t0, t1);
1029 tcg_temp_free_i32(t0);
1030 tcg_temp_free_i32(t1);
1031 }
1032 }
1033
1034 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1035 {
1036 if (TCG_TARGET_HAS_bswap32_i32) {
1037 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
1038 } else {
1039 TCGv_i32 t0 = tcg_temp_new_i32();
1040 TCGv_i32 t1 = tcg_temp_new_i32();
1041 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
1042
1043 /* arg = abcd */
1044 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */
1045 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */
1046 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */
1047 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */
1048 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */
1049
1050 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */
1051 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */
1052 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */
1053
1054 tcg_temp_free_i32(t0);
1055 tcg_temp_free_i32(t1);
1056 }
1057 }
1058
1059 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg)
1060 {
1061 /* Swapping 2 16-bit elements is a rotate. */
1062 tcg_gen_rotli_i32(ret, arg, 16);
1063 }
1064
1065 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1066 {
1067 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1068 }
1069
1070 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1071 {
1072 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1073 }
1074
1075 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1076 {
1077 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1078 }
1079
1080 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1081 {
1082 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1083 }
1084
1085 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1086 {
1087 TCGv_i32 t = tcg_temp_new_i32();
1088
1089 tcg_gen_sari_i32(t, a, 31);
1090 tcg_gen_xor_i32(ret, a, t);
1091 tcg_gen_sub_i32(ret, ret, t);
1092 tcg_temp_free_i32(t);
1093 }
1094
1095 /* 64-bit ops */
1096
1097 #if TCG_TARGET_REG_BITS == 32
1098 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1099
1100 void tcg_gen_discard_i64(TCGv_i64 arg)
1101 {
1102 tcg_gen_discard_i32(TCGV_LOW(arg));
1103 tcg_gen_discard_i32(TCGV_HIGH(arg));
1104 }
1105
1106 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1107 {
1108 TCGTemp *ts = tcgv_i64_temp(arg);
1109
1110 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1111 if (ts->kind == TEMP_CONST) {
1112 tcg_gen_movi_i64(ret, ts->val);
1113 } else {
1114 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1115 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1116 }
1117 }
1118
1119 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1120 {
1121 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1122 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1123 }
1124
1125 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1126 {
1127 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1128 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1129 }
1130
1131 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1132 {
1133 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1134 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1135 }
1136
1137 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1138 {
1139 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1140 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1141 }
1142
1143 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1144 {
1145 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1146 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1147 }
1148
1149 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1150 {
1151 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1152 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1153 }
1154
1155 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1156 {
1157 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1158 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1159 }
1160
1161 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1162 {
1163 /* Since arg2 and ret have different types,
1164 they cannot be the same temporary */
1165 #if HOST_BIG_ENDIAN
1166 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1167 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1168 #else
1169 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1170 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1171 #endif
1172 }
1173
1174 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1175 {
1176 #if HOST_BIG_ENDIAN
1177 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1178 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1179 #else
1180 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1181 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1182 #endif
1183 }
1184
1185 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1186 {
1187 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1188 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1189 }
1190
1191 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1192 {
1193 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1194 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1195 }
1196
1197 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1198 {
1199 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1200 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1201 }
1202
1203 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1204 {
1205 gen_helper_shl_i64(ret, arg1, arg2);
1206 }
1207
1208 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1209 {
1210 gen_helper_shr_i64(ret, arg1, arg2);
1211 }
1212
1213 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1214 {
1215 gen_helper_sar_i64(ret, arg1, arg2);
1216 }
1217
1218 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1219 {
1220 TCGv_i64 t0;
1221 TCGv_i32 t1;
1222
1223 t0 = tcg_temp_new_i64();
1224 t1 = tcg_temp_new_i32();
1225
1226 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1227 TCGV_LOW(arg1), TCGV_LOW(arg2));
1228
1229 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1230 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1231 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1232 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1233
1234 tcg_gen_mov_i64(ret, t0);
1235 tcg_temp_free_i64(t0);
1236 tcg_temp_free_i32(t1);
1237 }
1238
1239 #else
1240
1241 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1242 {
1243 tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1244 }
1245
1246 #endif /* TCG_TARGET_REG_SIZE == 32 */
1247
1248 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1249 {
1250 /* some cases can be optimized here */
1251 if (arg2 == 0) {
1252 tcg_gen_mov_i64(ret, arg1);
1253 } else if (TCG_TARGET_REG_BITS == 64) {
1254 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
1255 } else {
1256 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1257 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1258 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1259 }
1260 }
1261
1262 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1263 {
1264 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1265 /* Don't recurse with tcg_gen_neg_i64. */
1266 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1267 } else if (TCG_TARGET_REG_BITS == 64) {
1268 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
1269 } else {
1270 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1271 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
1272 TCGV_LOW(arg2), TCGV_HIGH(arg2));
1273 }
1274 }
1275
1276 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1277 {
1278 /* some cases can be optimized here */
1279 if (arg2 == 0) {
1280 tcg_gen_mov_i64(ret, arg1);
1281 } else if (TCG_TARGET_REG_BITS == 64) {
1282 tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2));
1283 } else {
1284 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1285 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1286 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1287 }
1288 }
1289
1290 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1291 {
1292 if (TCG_TARGET_REG_BITS == 32) {
1293 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1294 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1295 return;
1296 }
1297
1298 /* Some cases can be optimized here. */
1299 switch (arg2) {
1300 case 0:
1301 tcg_gen_movi_i64(ret, 0);
1302 return;
1303 case -1:
1304 tcg_gen_mov_i64(ret, arg1);
1305 return;
1306 case 0xff:
1307 /* Don't recurse with tcg_gen_ext8u_i64. */
1308 if (TCG_TARGET_HAS_ext8u_i64) {
1309 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1310 return;
1311 }
1312 break;
1313 case 0xffff:
1314 if (TCG_TARGET_HAS_ext16u_i64) {
1315 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1316 return;
1317 }
1318 break;
1319 case 0xffffffffu:
1320 if (TCG_TARGET_HAS_ext32u_i64) {
1321 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1322 return;
1323 }
1324 break;
1325 }
1326
1327 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
1328 }
1329
1330 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1331 {
1332 if (TCG_TARGET_REG_BITS == 32) {
1333 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1334 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1335 return;
1336 }
1337 /* Some cases can be optimized here. */
1338 if (arg2 == -1) {
1339 tcg_gen_movi_i64(ret, -1);
1340 } else if (arg2 == 0) {
1341 tcg_gen_mov_i64(ret, arg1);
1342 } else {
1343 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
1344 }
1345 }
1346
1347 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1348 {
1349 if (TCG_TARGET_REG_BITS == 32) {
1350 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1351 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1352 return;
1353 }
1354 /* Some cases can be optimized here. */
1355 if (arg2 == 0) {
1356 tcg_gen_mov_i64(ret, arg1);
1357 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1358 /* Don't recurse with tcg_gen_not_i64. */
1359 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1360 } else {
1361 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
1362 }
1363 }
1364
1365 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1366 unsigned c, bool right, bool arith)
1367 {
1368 tcg_debug_assert(c < 64);
1369 if (c == 0) {
1370 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1371 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1372 } else if (c >= 32) {
1373 c -= 32;
1374 if (right) {
1375 if (arith) {
1376 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1377 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1378 } else {
1379 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1380 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1381 }
1382 } else {
1383 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1384 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1385 }
1386 } else if (right) {
1387 if (TCG_TARGET_HAS_extract2_i32) {
1388 tcg_gen_extract2_i32(TCGV_LOW(ret),
1389 TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1390 } else {
1391 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1392 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1393 TCGV_HIGH(arg1), 32 - c, c);
1394 }
1395 if (arith) {
1396 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1397 } else {
1398 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1399 }
1400 } else {
1401 if (TCG_TARGET_HAS_extract2_i32) {
1402 tcg_gen_extract2_i32(TCGV_HIGH(ret),
1403 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
1404 } else {
1405 TCGv_i32 t0 = tcg_temp_new_i32();
1406 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1407 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1408 TCGV_HIGH(arg1), c, 32 - c);
1409 tcg_temp_free_i32(t0);
1410 }
1411 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1412 }
1413 }
1414
1415 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1416 {
1417 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1418 if (TCG_TARGET_REG_BITS == 32) {
1419 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1420 } else if (arg2 == 0) {
1421 tcg_gen_mov_i64(ret, arg1);
1422 } else {
1423 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
1424 }
1425 }
1426
1427 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1428 {
1429 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1430 if (TCG_TARGET_REG_BITS == 32) {
1431 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1432 } else if (arg2 == 0) {
1433 tcg_gen_mov_i64(ret, arg1);
1434 } else {
1435 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
1436 }
1437 }
1438
1439 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1440 {
1441 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1442 if (TCG_TARGET_REG_BITS == 32) {
1443 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1444 } else if (arg2 == 0) {
1445 tcg_gen_mov_i64(ret, arg1);
1446 } else {
1447 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
1448 }
1449 }
1450
1451 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1452 {
1453 if (cond == TCG_COND_ALWAYS) {
1454 tcg_gen_br(l);
1455 } else if (cond != TCG_COND_NEVER) {
1456 l->refs++;
1457 if (TCG_TARGET_REG_BITS == 32) {
1458 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1459 TCGV_HIGH(arg1), TCGV_LOW(arg2),
1460 TCGV_HIGH(arg2), cond, label_arg(l));
1461 } else {
1462 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1463 label_arg(l));
1464 }
1465 }
1466 }
1467
1468 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1469 {
1470 if (TCG_TARGET_REG_BITS == 64) {
1471 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1472 } else if (cond == TCG_COND_ALWAYS) {
1473 tcg_gen_br(l);
1474 } else if (cond != TCG_COND_NEVER) {
1475 l->refs++;
1476 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
1477 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1478 tcg_constant_i32(arg2),
1479 tcg_constant_i32(arg2 >> 32),
1480 cond, label_arg(l));
1481 }
1482 }
1483
1484 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1485 TCGv_i64 arg1, TCGv_i64 arg2)
1486 {
1487 if (cond == TCG_COND_ALWAYS) {
1488 tcg_gen_movi_i64(ret, 1);
1489 } else if (cond == TCG_COND_NEVER) {
1490 tcg_gen_movi_i64(ret, 0);
1491 } else {
1492 if (TCG_TARGET_REG_BITS == 32) {
1493 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1494 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1495 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1496 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1497 } else {
1498 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1499 }
1500 }
1501 }
1502
1503 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1504 TCGv_i64 arg1, int64_t arg2)
1505 {
1506 if (TCG_TARGET_REG_BITS == 64) {
1507 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1508 } else if (cond == TCG_COND_ALWAYS) {
1509 tcg_gen_movi_i64(ret, 1);
1510 } else if (cond == TCG_COND_NEVER) {
1511 tcg_gen_movi_i64(ret, 0);
1512 } else {
1513 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1514 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1515 tcg_constant_i32(arg2),
1516 tcg_constant_i32(arg2 >> 32), cond);
1517 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1518 }
1519 }
1520
1521 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1522 {
1523 if (arg2 == 0) {
1524 tcg_gen_movi_i64(ret, 0);
1525 } else if (is_power_of_2(arg2)) {
1526 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1527 } else {
1528 TCGv_i64 t0 = tcg_const_i64(arg2);
1529 tcg_gen_mul_i64(ret, arg1, t0);
1530 tcg_temp_free_i64(t0);
1531 }
1532 }
1533
1534 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1535 {
1536 if (TCG_TARGET_HAS_div_i64) {
1537 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1538 } else if (TCG_TARGET_HAS_div2_i64) {
1539 TCGv_i64 t0 = tcg_temp_new_i64();
1540 tcg_gen_sari_i64(t0, arg1, 63);
1541 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1542 tcg_temp_free_i64(t0);
1543 } else {
1544 gen_helper_div_i64(ret, arg1, arg2);
1545 }
1546 }
1547
1548 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1549 {
1550 if (TCG_TARGET_HAS_rem_i64) {
1551 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1552 } else if (TCG_TARGET_HAS_div_i64) {
1553 TCGv_i64 t0 = tcg_temp_new_i64();
1554 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1555 tcg_gen_mul_i64(t0, t0, arg2);
1556 tcg_gen_sub_i64(ret, arg1, t0);
1557 tcg_temp_free_i64(t0);
1558 } else if (TCG_TARGET_HAS_div2_i64) {
1559 TCGv_i64 t0 = tcg_temp_new_i64();
1560 tcg_gen_sari_i64(t0, arg1, 63);
1561 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1562 tcg_temp_free_i64(t0);
1563 } else {
1564 gen_helper_rem_i64(ret, arg1, arg2);
1565 }
1566 }
1567
1568 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1569 {
1570 if (TCG_TARGET_HAS_div_i64) {
1571 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1572 } else if (TCG_TARGET_HAS_div2_i64) {
1573 TCGv_i64 t0 = tcg_temp_new_i64();
1574 tcg_gen_movi_i64(t0, 0);
1575 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1576 tcg_temp_free_i64(t0);
1577 } else {
1578 gen_helper_divu_i64(ret, arg1, arg2);
1579 }
1580 }
1581
1582 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1583 {
1584 if (TCG_TARGET_HAS_rem_i64) {
1585 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1586 } else if (TCG_TARGET_HAS_div_i64) {
1587 TCGv_i64 t0 = tcg_temp_new_i64();
1588 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1589 tcg_gen_mul_i64(t0, t0, arg2);
1590 tcg_gen_sub_i64(ret, arg1, t0);
1591 tcg_temp_free_i64(t0);
1592 } else if (TCG_TARGET_HAS_div2_i64) {
1593 TCGv_i64 t0 = tcg_temp_new_i64();
1594 tcg_gen_movi_i64(t0, 0);
1595 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1596 tcg_temp_free_i64(t0);
1597 } else {
1598 gen_helper_remu_i64(ret, arg1, arg2);
1599 }
1600 }
1601
1602 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1603 {
1604 if (TCG_TARGET_REG_BITS == 32) {
1605 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1606 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1607 } else if (TCG_TARGET_HAS_ext8s_i64) {
1608 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1609 } else {
1610 tcg_gen_shli_i64(ret, arg, 56);
1611 tcg_gen_sari_i64(ret, ret, 56);
1612 }
1613 }
1614
1615 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1616 {
1617 if (TCG_TARGET_REG_BITS == 32) {
1618 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1619 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1620 } else if (TCG_TARGET_HAS_ext16s_i64) {
1621 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1622 } else {
1623 tcg_gen_shli_i64(ret, arg, 48);
1624 tcg_gen_sari_i64(ret, ret, 48);
1625 }
1626 }
1627
1628 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1629 {
1630 if (TCG_TARGET_REG_BITS == 32) {
1631 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1632 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1633 } else if (TCG_TARGET_HAS_ext32s_i64) {
1634 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1635 } else {
1636 tcg_gen_shli_i64(ret, arg, 32);
1637 tcg_gen_sari_i64(ret, ret, 32);
1638 }
1639 }
1640
1641 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1642 {
1643 if (TCG_TARGET_REG_BITS == 32) {
1644 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1645 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1646 } else if (TCG_TARGET_HAS_ext8u_i64) {
1647 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1648 } else {
1649 tcg_gen_andi_i64(ret, arg, 0xffu);
1650 }
1651 }
1652
1653 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1654 {
1655 if (TCG_TARGET_REG_BITS == 32) {
1656 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1657 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1658 } else if (TCG_TARGET_HAS_ext16u_i64) {
1659 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1660 } else {
1661 tcg_gen_andi_i64(ret, arg, 0xffffu);
1662 }
1663 }
1664
1665 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1666 {
1667 if (TCG_TARGET_REG_BITS == 32) {
1668 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1669 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1670 } else if (TCG_TARGET_HAS_ext32u_i64) {
1671 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1672 } else {
1673 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1674 }
1675 }
1676
1677 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1678 {
1679 /* Only one extension flag may be present. */
1680 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1681
1682 if (TCG_TARGET_REG_BITS == 32) {
1683 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
1684 if (flags & TCG_BSWAP_OS) {
1685 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1686 } else {
1687 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1688 }
1689 } else if (TCG_TARGET_HAS_bswap16_i64) {
1690 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
1691 } else {
1692 TCGv_i64 t0 = tcg_temp_new_i64();
1693 TCGv_i64 t1 = tcg_temp_new_i64();
1694
1695 tcg_gen_shri_i64(t0, arg, 8);
1696 if (!(flags & TCG_BSWAP_IZ)) {
1697 tcg_gen_ext8u_i64(t0, t0);
1698 }
1699
1700 if (flags & TCG_BSWAP_OS) {
1701 tcg_gen_shli_i64(t1, arg, 56);
1702 tcg_gen_sari_i64(t1, t1, 48);
1703 } else if (flags & TCG_BSWAP_OZ) {
1704 tcg_gen_ext8u_i64(t1, arg);
1705 tcg_gen_shli_i64(t1, t1, 8);
1706 } else {
1707 tcg_gen_shli_i64(t1, arg, 8);
1708 }
1709
1710 tcg_gen_or_i64(ret, t0, t1);
1711 tcg_temp_free_i64(t0);
1712 tcg_temp_free_i64(t1);
1713 }
1714 }
1715
1716 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1717 {
1718 /* Only one extension flag may be present. */
1719 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1720
1721 if (TCG_TARGET_REG_BITS == 32) {
1722 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1723 if (flags & TCG_BSWAP_OS) {
1724 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1725 } else {
1726 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1727 }
1728 } else if (TCG_TARGET_HAS_bswap32_i64) {
1729 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
1730 } else {
1731 TCGv_i64 t0 = tcg_temp_new_i64();
1732 TCGv_i64 t1 = tcg_temp_new_i64();
1733 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
1734
1735 /* arg = xxxxabcd */
1736 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */
1737 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */
1738 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */
1739 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */
1740 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */
1741
1742 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1743 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1744 if (flags & TCG_BSWAP_OS) {
1745 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */
1746 } else {
1747 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1748 }
1749 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba */
1750
1751 tcg_temp_free_i64(t0);
1752 tcg_temp_free_i64(t1);
1753 }
1754 }
1755
1756 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1757 {
1758 if (TCG_TARGET_REG_BITS == 32) {
1759 TCGv_i32 t0, t1;
1760 t0 = tcg_temp_new_i32();
1761 t1 = tcg_temp_new_i32();
1762
1763 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1764 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1765 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1766 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1767 tcg_temp_free_i32(t0);
1768 tcg_temp_free_i32(t1);
1769 } else if (TCG_TARGET_HAS_bswap64_i64) {
1770 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
1771 } else {
1772 TCGv_i64 t0 = tcg_temp_new_i64();
1773 TCGv_i64 t1 = tcg_temp_new_i64();
1774 TCGv_i64 t2 = tcg_temp_new_i64();
1775
1776 /* arg = abcdefgh */
1777 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1778 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */
1779 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */
1780 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */
1781 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */
1782 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */
1783
1784 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1785 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */
1786 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */
1787 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */
1788 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */
1789 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */
1790
1791 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */
1792 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */
1793 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */
1794
1795 tcg_temp_free_i64(t0);
1796 tcg_temp_free_i64(t1);
1797 tcg_temp_free_i64(t2);
1798 }
1799 }
1800
1801 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1802 {
1803 uint64_t m = 0x0000ffff0000ffffull;
1804 TCGv_i64 t0 = tcg_temp_new_i64();
1805 TCGv_i64 t1 = tcg_temp_new_i64();
1806
1807 /* See include/qemu/bitops.h, hswap64. */
1808 tcg_gen_rotli_i64(t1, arg, 32);
1809 tcg_gen_andi_i64(t0, t1, m);
1810 tcg_gen_shli_i64(t0, t0, 16);
1811 tcg_gen_shri_i64(t1, t1, 16);
1812 tcg_gen_andi_i64(t1, t1, m);
1813 tcg_gen_or_i64(ret, t0, t1);
1814
1815 tcg_temp_free_i64(t0);
1816 tcg_temp_free_i64(t1);
1817 }
1818
1819 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1820 {
1821 /* Swapping 2 32-bit elements is a rotate. */
1822 tcg_gen_rotli_i64(ret, arg, 32);
1823 }
1824
1825 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1826 {
1827 if (TCG_TARGET_REG_BITS == 32) {
1828 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1829 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1830 } else if (TCG_TARGET_HAS_not_i64) {
1831 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1832 } else {
1833 tcg_gen_xori_i64(ret, arg, -1);
1834 }
1835 }
1836
1837 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1838 {
1839 if (TCG_TARGET_REG_BITS == 32) {
1840 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1841 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1842 } else if (TCG_TARGET_HAS_andc_i64) {
1843 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1844 } else {
1845 TCGv_i64 t0 = tcg_temp_new_i64();
1846 tcg_gen_not_i64(t0, arg2);
1847 tcg_gen_and_i64(ret, arg1, t0);
1848 tcg_temp_free_i64(t0);
1849 }
1850 }
1851
1852 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1853 {
1854 if (TCG_TARGET_REG_BITS == 32) {
1855 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1856 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1857 } else if (TCG_TARGET_HAS_eqv_i64) {
1858 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1859 } else {
1860 tcg_gen_xor_i64(ret, arg1, arg2);
1861 tcg_gen_not_i64(ret, ret);
1862 }
1863 }
1864
1865 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1866 {
1867 if (TCG_TARGET_REG_BITS == 32) {
1868 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1869 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1870 } else if (TCG_TARGET_HAS_nand_i64) {
1871 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1872 } else {
1873 tcg_gen_and_i64(ret, arg1, arg2);
1874 tcg_gen_not_i64(ret, ret);
1875 }
1876 }
1877
1878 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1879 {
1880 if (TCG_TARGET_REG_BITS == 32) {
1881 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1882 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1883 } else if (TCG_TARGET_HAS_nor_i64) {
1884 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1885 } else {
1886 tcg_gen_or_i64(ret, arg1, arg2);
1887 tcg_gen_not_i64(ret, ret);
1888 }
1889 }
1890
1891 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1892 {
1893 if (TCG_TARGET_REG_BITS == 32) {
1894 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1895 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1896 } else if (TCG_TARGET_HAS_orc_i64) {
1897 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1898 } else {
1899 TCGv_i64 t0 = tcg_temp_new_i64();
1900 tcg_gen_not_i64(t0, arg2);
1901 tcg_gen_or_i64(ret, arg1, t0);
1902 tcg_temp_free_i64(t0);
1903 }
1904 }
1905
1906 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1907 {
1908 if (TCG_TARGET_HAS_clz_i64) {
1909 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1910 } else {
1911 gen_helper_clz_i64(ret, arg1, arg2);
1912 }
1913 }
1914
1915 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1916 {
1917 if (TCG_TARGET_REG_BITS == 32
1918 && TCG_TARGET_HAS_clz_i32
1919 && arg2 <= 0xffffffffu) {
1920 TCGv_i32 t = tcg_temp_new_i32();
1921 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
1922 tcg_gen_addi_i32(t, t, 32);
1923 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1924 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1925 tcg_temp_free_i32(t);
1926 } else {
1927 TCGv_i64 t0 = tcg_const_i64(arg2);
1928 tcg_gen_clz_i64(ret, arg1, t0);
1929 tcg_temp_free_i64(t0);
1930 }
1931 }
1932
1933 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1934 {
1935 if (TCG_TARGET_HAS_ctz_i64) {
1936 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1937 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1938 TCGv_i64 z, t = tcg_temp_new_i64();
1939
1940 if (TCG_TARGET_HAS_ctpop_i64) {
1941 tcg_gen_subi_i64(t, arg1, 1);
1942 tcg_gen_andc_i64(t, t, arg1);
1943 tcg_gen_ctpop_i64(t, t);
1944 } else {
1945 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1946 tcg_gen_neg_i64(t, arg1);
1947 tcg_gen_and_i64(t, t, arg1);
1948 tcg_gen_clzi_i64(t, t, 64);
1949 tcg_gen_xori_i64(t, t, 63);
1950 }
1951 z = tcg_constant_i64(0);
1952 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1953 tcg_temp_free_i64(t);
1954 tcg_temp_free_i64(z);
1955 } else {
1956 gen_helper_ctz_i64(ret, arg1, arg2);
1957 }
1958 }
1959
1960 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1961 {
1962 if (TCG_TARGET_REG_BITS == 32
1963 && TCG_TARGET_HAS_ctz_i32
1964 && arg2 <= 0xffffffffu) {
1965 TCGv_i32 t32 = tcg_temp_new_i32();
1966 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
1967 tcg_gen_addi_i32(t32, t32, 32);
1968 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1969 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1970 tcg_temp_free_i32(t32);
1971 } else if (!TCG_TARGET_HAS_ctz_i64
1972 && TCG_TARGET_HAS_ctpop_i64
1973 && arg2 == 64) {
1974 /* This equivalence has the advantage of not requiring a fixup. */
1975 TCGv_i64 t = tcg_temp_new_i64();
1976 tcg_gen_subi_i64(t, arg1, 1);
1977 tcg_gen_andc_i64(t, t, arg1);
1978 tcg_gen_ctpop_i64(ret, t);
1979 tcg_temp_free_i64(t);
1980 } else {
1981 TCGv_i64 t0 = tcg_const_i64(arg2);
1982 tcg_gen_ctz_i64(ret, arg1, t0);
1983 tcg_temp_free_i64(t0);
1984 }
1985 }
1986
1987 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1988 {
1989 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1990 TCGv_i64 t = tcg_temp_new_i64();
1991 tcg_gen_sari_i64(t, arg, 63);
1992 tcg_gen_xor_i64(t, t, arg);
1993 tcg_gen_clzi_i64(t, t, 64);
1994 tcg_gen_subi_i64(ret, t, 1);
1995 tcg_temp_free_i64(t);
1996 } else {
1997 gen_helper_clrsb_i64(ret, arg);
1998 }
1999 }
2000
2001 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
2002 {
2003 if (TCG_TARGET_HAS_ctpop_i64) {
2004 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
2005 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
2006 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2007 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2008 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
2009 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2010 } else {
2011 gen_helper_ctpop_i64(ret, arg1);
2012 }
2013 }
2014
2015 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2016 {
2017 if (TCG_TARGET_HAS_rot_i64) {
2018 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
2019 } else {
2020 TCGv_i64 t0, t1;
2021 t0 = tcg_temp_new_i64();
2022 t1 = tcg_temp_new_i64();
2023 tcg_gen_shl_i64(t0, arg1, arg2);
2024 tcg_gen_subfi_i64(t1, 64, arg2);
2025 tcg_gen_shr_i64(t1, arg1, t1);
2026 tcg_gen_or_i64(ret, t0, t1);
2027 tcg_temp_free_i64(t0);
2028 tcg_temp_free_i64(t1);
2029 }
2030 }
2031
2032 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2033 {
2034 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2035 /* some cases can be optimized here */
2036 if (arg2 == 0) {
2037 tcg_gen_mov_i64(ret, arg1);
2038 } else if (TCG_TARGET_HAS_rot_i64) {
2039 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
2040 } else {
2041 TCGv_i64 t0, t1;
2042 t0 = tcg_temp_new_i64();
2043 t1 = tcg_temp_new_i64();
2044 tcg_gen_shli_i64(t0, arg1, arg2);
2045 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
2046 tcg_gen_or_i64(ret, t0, t1);
2047 tcg_temp_free_i64(t0);
2048 tcg_temp_free_i64(t1);
2049 }
2050 }
2051
2052 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2053 {
2054 if (TCG_TARGET_HAS_rot_i64) {
2055 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
2056 } else {
2057 TCGv_i64 t0, t1;
2058 t0 = tcg_temp_new_i64();
2059 t1 = tcg_temp_new_i64();
2060 tcg_gen_shr_i64(t0, arg1, arg2);
2061 tcg_gen_subfi_i64(t1, 64, arg2);
2062 tcg_gen_shl_i64(t1, arg1, t1);
2063 tcg_gen_or_i64(ret, t0, t1);
2064 tcg_temp_free_i64(t0);
2065 tcg_temp_free_i64(t1);
2066 }
2067 }
2068
2069 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2070 {
2071 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2072 /* some cases can be optimized here */
2073 if (arg2 == 0) {
2074 tcg_gen_mov_i64(ret, arg1);
2075 } else {
2076 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2077 }
2078 }
2079
2080 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2081 unsigned int ofs, unsigned int len)
2082 {
2083 uint64_t mask;
2084 TCGv_i64 t1;
2085
2086 tcg_debug_assert(ofs < 64);
2087 tcg_debug_assert(len > 0);
2088 tcg_debug_assert(len <= 64);
2089 tcg_debug_assert(ofs + len <= 64);
2090
2091 if (len == 64) {
2092 tcg_gen_mov_i64(ret, arg2);
2093 return;
2094 }
2095 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2096 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2097 return;
2098 }
2099
2100 if (TCG_TARGET_REG_BITS == 32) {
2101 if (ofs >= 32) {
2102 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2103 TCGV_LOW(arg2), ofs - 32, len);
2104 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2105 return;
2106 }
2107 if (ofs + len <= 32) {
2108 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2109 TCGV_LOW(arg2), ofs, len);
2110 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2111 return;
2112 }
2113 }
2114
2115 t1 = tcg_temp_new_i64();
2116
2117 if (TCG_TARGET_HAS_extract2_i64) {
2118 if (ofs + len == 64) {
2119 tcg_gen_shli_i64(t1, arg1, len);
2120 tcg_gen_extract2_i64(ret, t1, arg2, len);
2121 goto done;
2122 }
2123 if (ofs == 0) {
2124 tcg_gen_extract2_i64(ret, arg1, arg2, len);
2125 tcg_gen_rotli_i64(ret, ret, len);
2126 goto done;
2127 }
2128 }
2129
2130 mask = (1ull << len) - 1;
2131 if (ofs + len < 64) {
2132 tcg_gen_andi_i64(t1, arg2, mask);
2133 tcg_gen_shli_i64(t1, t1, ofs);
2134 } else {
2135 tcg_gen_shli_i64(t1, arg2, ofs);
2136 }
2137 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2138 tcg_gen_or_i64(ret, ret, t1);
2139 done:
2140 tcg_temp_free_i64(t1);
2141 }
2142
2143 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2144 unsigned int ofs, unsigned int len)
2145 {
2146 tcg_debug_assert(ofs < 64);
2147 tcg_debug_assert(len > 0);
2148 tcg_debug_assert(len <= 64);
2149 tcg_debug_assert(ofs + len <= 64);
2150
2151 if (ofs + len == 64) {
2152 tcg_gen_shli_i64(ret, arg, ofs);
2153 } else if (ofs == 0) {
2154 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2155 } else if (TCG_TARGET_HAS_deposit_i64
2156 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2157 TCGv_i64 zero = tcg_constant_i64(0);
2158 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2159 } else {
2160 if (TCG_TARGET_REG_BITS == 32) {
2161 if (ofs >= 32) {
2162 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2163 ofs - 32, len);
2164 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2165 return;
2166 }
2167 if (ofs + len <= 32) {
2168 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2169 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2170 return;
2171 }
2172 }
2173 /* To help two-operand hosts we prefer to zero-extend first,
2174 which allows ARG to stay live. */
2175 switch (len) {
2176 case 32:
2177 if (TCG_TARGET_HAS_ext32u_i64) {
2178 tcg_gen_ext32u_i64(ret, arg);
2179 tcg_gen_shli_i64(ret, ret, ofs);
2180 return;
2181 }
2182 break;
2183 case 16:
2184 if (TCG_TARGET_HAS_ext16u_i64) {
2185 tcg_gen_ext16u_i64(ret, arg);
2186 tcg_gen_shli_i64(ret, ret, ofs);
2187 return;
2188 }
2189 break;
2190 case 8:
2191 if (TCG_TARGET_HAS_ext8u_i64) {
2192 tcg_gen_ext8u_i64(ret, arg);
2193 tcg_gen_shli_i64(ret, ret, ofs);
2194 return;
2195 }
2196 break;
2197 }
2198 /* Otherwise prefer zero-extension over AND for code size. */
2199 switch (ofs + len) {
2200 case 32:
2201 if (TCG_TARGET_HAS_ext32u_i64) {
2202 tcg_gen_shli_i64(ret, arg, ofs);
2203 tcg_gen_ext32u_i64(ret, ret);
2204 return;
2205 }
2206 break;
2207 case 16:
2208 if (TCG_TARGET_HAS_ext16u_i64) {
2209 tcg_gen_shli_i64(ret, arg, ofs);
2210 tcg_gen_ext16u_i64(ret, ret);
2211 return;
2212 }
2213 break;
2214 case 8:
2215 if (TCG_TARGET_HAS_ext8u_i64) {
2216 tcg_gen_shli_i64(ret, arg, ofs);
2217 tcg_gen_ext8u_i64(ret, ret);
2218 return;
2219 }
2220 break;
2221 }
2222 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2223 tcg_gen_shli_i64(ret, ret, ofs);
2224 }
2225 }
2226
2227 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2228 unsigned int ofs, unsigned int len)
2229 {
2230 tcg_debug_assert(ofs < 64);
2231 tcg_debug_assert(len > 0);
2232 tcg_debug_assert(len <= 64);
2233 tcg_debug_assert(ofs + len <= 64);
2234
2235 /* Canonicalize certain special cases, even if extract is supported. */
2236 if (ofs + len == 64) {
2237 tcg_gen_shri_i64(ret, arg, 64 - len);
2238 return;
2239 }
2240 if (ofs == 0) {
2241 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2242 return;
2243 }
2244
2245 if (TCG_TARGET_REG_BITS == 32) {
2246 /* Look for a 32-bit extract within one of the two words. */
2247 if (ofs >= 32) {
2248 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2249 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2250 return;
2251 }
2252 if (ofs + len <= 32) {
2253 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2254 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2255 return;
2256 }
2257 /* The field is split across two words. One double-word
2258 shift is better than two double-word shifts. */
2259 goto do_shift_and;
2260 }
2261
2262 if (TCG_TARGET_HAS_extract_i64
2263 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2264 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2265 return;
2266 }
2267
2268 /* Assume that zero-extension, if available, is cheaper than a shift. */
2269 switch (ofs + len) {
2270 case 32:
2271 if (TCG_TARGET_HAS_ext32u_i64) {
2272 tcg_gen_ext32u_i64(ret, arg);
2273 tcg_gen_shri_i64(ret, ret, ofs);
2274 return;
2275 }
2276 break;
2277 case 16:
2278 if (TCG_TARGET_HAS_ext16u_i64) {
2279 tcg_gen_ext16u_i64(ret, arg);
2280 tcg_gen_shri_i64(ret, ret, ofs);
2281 return;
2282 }
2283 break;
2284 case 8:
2285 if (TCG_TARGET_HAS_ext8u_i64) {
2286 tcg_gen_ext8u_i64(ret, arg);
2287 tcg_gen_shri_i64(ret, ret, ofs);
2288 return;
2289 }
2290 break;
2291 }
2292
2293 /* ??? Ideally we'd know what values are available for immediate AND.
2294 Assume that 8 bits are available, plus the special cases of 16 and 32,
2295 so that we get ext8u, ext16u, and ext32u. */
2296 switch (len) {
2297 case 1 ... 8: case 16: case 32:
2298 do_shift_and:
2299 tcg_gen_shri_i64(ret, arg, ofs);
2300 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2301 break;
2302 default:
2303 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2304 tcg_gen_shri_i64(ret, ret, 64 - len);
2305 break;
2306 }
2307 }
2308
2309 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2310 unsigned int ofs, unsigned int len)
2311 {
2312 tcg_debug_assert(ofs < 64);
2313 tcg_debug_assert(len > 0);
2314 tcg_debug_assert(len <= 64);
2315 tcg_debug_assert(ofs + len <= 64);
2316
2317 /* Canonicalize certain special cases, even if sextract is supported. */
2318 if (ofs + len == 64) {
2319 tcg_gen_sari_i64(ret, arg, 64 - len);
2320 return;
2321 }
2322 if (ofs == 0) {
2323 switch (len) {
2324 case 32:
2325 tcg_gen_ext32s_i64(ret, arg);
2326 return;
2327 case 16:
2328 tcg_gen_ext16s_i64(ret, arg);
2329 return;
2330 case 8:
2331 tcg_gen_ext8s_i64(ret, arg);
2332 return;
2333 }
2334 }
2335
2336 if (TCG_TARGET_REG_BITS == 32) {
2337 /* Look for a 32-bit extract within one of the two words. */
2338 if (ofs >= 32) {
2339 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2340 } else if (ofs + len <= 32) {
2341 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2342 } else if (ofs == 0) {
2343 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2344 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2345 return;
2346 } else if (len > 32) {
2347 TCGv_i32 t = tcg_temp_new_i32();
2348 /* Extract the bits for the high word normally. */
2349 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2350 /* Shift the field down for the low part. */
2351 tcg_gen_shri_i64(ret, arg, ofs);
2352 /* Overwrite the shift into the high part. */
2353 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2354 tcg_temp_free_i32(t);
2355 return;
2356 } else {
2357 /* Shift the field down for the low part, such that the
2358 field sits at the MSB. */
2359 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2360 /* Shift the field down from the MSB, sign extending. */
2361 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2362 }
2363 /* Sign-extend the field from 32 bits. */
2364 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2365 return;
2366 }
2367
2368 if (TCG_TARGET_HAS_sextract_i64
2369 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2370 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2371 return;
2372 }
2373
2374 /* Assume that sign-extension, if available, is cheaper than a shift. */
2375 switch (ofs + len) {
2376 case 32:
2377 if (TCG_TARGET_HAS_ext32s_i64) {
2378 tcg_gen_ext32s_i64(ret, arg);
2379 tcg_gen_sari_i64(ret, ret, ofs);
2380 return;
2381 }
2382 break;
2383 case 16:
2384 if (TCG_TARGET_HAS_ext16s_i64) {
2385 tcg_gen_ext16s_i64(ret, arg);
2386 tcg_gen_sari_i64(ret, ret, ofs);
2387 return;
2388 }
2389 break;
2390 case 8:
2391 if (TCG_TARGET_HAS_ext8s_i64) {
2392 tcg_gen_ext8s_i64(ret, arg);
2393 tcg_gen_sari_i64(ret, ret, ofs);
2394 return;
2395 }
2396 break;
2397 }
2398 switch (len) {
2399 case 32:
2400 if (TCG_TARGET_HAS_ext32s_i64) {
2401 tcg_gen_shri_i64(ret, arg, ofs);
2402 tcg_gen_ext32s_i64(ret, ret);
2403 return;
2404 }
2405 break;
2406 case 16:
2407 if (TCG_TARGET_HAS_ext16s_i64) {
2408 tcg_gen_shri_i64(ret, arg, ofs);
2409 tcg_gen_ext16s_i64(ret, ret);
2410 return;
2411 }
2412 break;
2413 case 8:
2414 if (TCG_TARGET_HAS_ext8s_i64) {
2415 tcg_gen_shri_i64(ret, arg, ofs);
2416 tcg_gen_ext8s_i64(ret, ret);
2417 return;
2418 }
2419 break;
2420 }
2421 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2422 tcg_gen_sari_i64(ret, ret, 64 - len);
2423 }
2424
2425 /*
2426 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2427 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2428 */
2429 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2430 unsigned int ofs)
2431 {
2432 tcg_debug_assert(ofs <= 64);
2433 if (ofs == 0) {
2434 tcg_gen_mov_i64(ret, al);
2435 } else if (ofs == 64) {
2436 tcg_gen_mov_i64(ret, ah);
2437 } else if (al == ah) {
2438 tcg_gen_rotri_i64(ret, al, ofs);
2439 } else if (TCG_TARGET_HAS_extract2_i64) {
2440 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2441 } else {
2442 TCGv_i64 t0 = tcg_temp_new_i64();
2443 tcg_gen_shri_i64(t0, al, ofs);
2444 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2445 tcg_temp_free_i64(t0);
2446 }
2447 }
2448
2449 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2450 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2451 {
2452 if (cond == TCG_COND_ALWAYS) {
2453 tcg_gen_mov_i64(ret, v1);
2454 } else if (cond == TCG_COND_NEVER) {
2455 tcg_gen_mov_i64(ret, v2);
2456 } else if (TCG_TARGET_REG_BITS == 32) {
2457 TCGv_i32 t0 = tcg_temp_new_i32();
2458 TCGv_i32 t1 = tcg_temp_new_i32();
2459 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2460 TCGV_LOW(c1), TCGV_HIGH(c1),
2461 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2462
2463 if (TCG_TARGET_HAS_movcond_i32) {
2464 tcg_gen_movi_i32(t1, 0);
2465 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2466 TCGV_LOW(v1), TCGV_LOW(v2));
2467 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2468 TCGV_HIGH(v1), TCGV_HIGH(v2));
2469 } else {
2470 tcg_gen_neg_i32(t0, t0);
2471
2472 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2473 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2474 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2475
2476 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2477 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2478 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2479 }
2480 tcg_temp_free_i32(t0);
2481 tcg_temp_free_i32(t1);
2482 } else if (TCG_TARGET_HAS_movcond_i64) {
2483 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2484 } else {
2485 TCGv_i64 t0 = tcg_temp_new_i64();
2486 TCGv_i64 t1 = tcg_temp_new_i64();
2487 tcg_gen_setcond_i64(cond, t0, c1, c2);
2488 tcg_gen_neg_i64(t0, t0);
2489 tcg_gen_and_i64(t1, v1, t0);
2490 tcg_gen_andc_i64(ret, v2, t0);
2491 tcg_gen_or_i64(ret, ret, t1);
2492 tcg_temp_free_i64(t0);
2493 tcg_temp_free_i64(t1);
2494 }
2495 }
2496
2497 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2498 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2499 {
2500 if (TCG_TARGET_HAS_add2_i64) {
2501 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2502 } else {
2503 TCGv_i64 t0 = tcg_temp_new_i64();
2504 TCGv_i64 t1 = tcg_temp_new_i64();
2505 tcg_gen_add_i64(t0, al, bl);
2506 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2507 tcg_gen_add_i64(rh, ah, bh);
2508 tcg_gen_add_i64(rh, rh, t1);
2509 tcg_gen_mov_i64(rl, t0);
2510 tcg_temp_free_i64(t0);
2511 tcg_temp_free_i64(t1);
2512 }
2513 }
2514
2515 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2516 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2517 {
2518 if (TCG_TARGET_HAS_sub2_i64) {
2519 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2520 } else {
2521 TCGv_i64 t0 = tcg_temp_new_i64();
2522 TCGv_i64 t1 = tcg_temp_new_i64();
2523 tcg_gen_sub_i64(t0, al, bl);
2524 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2525 tcg_gen_sub_i64(rh, ah, bh);
2526 tcg_gen_sub_i64(rh, rh, t1);
2527 tcg_gen_mov_i64(rl, t0);
2528 tcg_temp_free_i64(t0);
2529 tcg_temp_free_i64(t1);
2530 }
2531 }
2532
2533 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2534 {
2535 if (TCG_TARGET_HAS_mulu2_i64) {
2536 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2537 } else if (TCG_TARGET_HAS_muluh_i64) {
2538 TCGv_i64 t = tcg_temp_new_i64();
2539 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2540 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2541 tcg_gen_mov_i64(rl, t);
2542 tcg_temp_free_i64(t);
2543 } else {
2544 TCGv_i64 t0 = tcg_temp_new_i64();
2545 tcg_gen_mul_i64(t0, arg1, arg2);
2546 gen_helper_muluh_i64(rh, arg1, arg2);
2547 tcg_gen_mov_i64(rl, t0);
2548 tcg_temp_free_i64(t0);
2549 }
2550 }
2551
2552 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2553 {
2554 if (TCG_TARGET_HAS_muls2_i64) {
2555 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2556 } else if (TCG_TARGET_HAS_mulsh_i64) {
2557 TCGv_i64 t = tcg_temp_new_i64();
2558 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2559 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2560 tcg_gen_mov_i64(rl, t);
2561 tcg_temp_free_i64(t);
2562 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2563 TCGv_i64 t0 = tcg_temp_new_i64();
2564 TCGv_i64 t1 = tcg_temp_new_i64();
2565 TCGv_i64 t2 = tcg_temp_new_i64();
2566 TCGv_i64 t3 = tcg_temp_new_i64();
2567 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2568 /* Adjust for negative inputs. */
2569 tcg_gen_sari_i64(t2, arg1, 63);
2570 tcg_gen_sari_i64(t3, arg2, 63);
2571 tcg_gen_and_i64(t2, t2, arg2);
2572 tcg_gen_and_i64(t3, t3, arg1);
2573 tcg_gen_sub_i64(rh, t1, t2);
2574 tcg_gen_sub_i64(rh, rh, t3);
2575 tcg_gen_mov_i64(rl, t0);
2576 tcg_temp_free_i64(t0);
2577 tcg_temp_free_i64(t1);
2578 tcg_temp_free_i64(t2);
2579 tcg_temp_free_i64(t3);
2580 } else {
2581 TCGv_i64 t0 = tcg_temp_new_i64();
2582 tcg_gen_mul_i64(t0, arg1, arg2);
2583 gen_helper_mulsh_i64(rh, arg1, arg2);
2584 tcg_gen_mov_i64(rl, t0);
2585 tcg_temp_free_i64(t0);
2586 }
2587 }
2588
2589 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2590 {
2591 TCGv_i64 t0 = tcg_temp_new_i64();
2592 TCGv_i64 t1 = tcg_temp_new_i64();
2593 TCGv_i64 t2 = tcg_temp_new_i64();
2594 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2595 /* Adjust for negative input for the signed arg1. */
2596 tcg_gen_sari_i64(t2, arg1, 63);
2597 tcg_gen_and_i64(t2, t2, arg2);
2598 tcg_gen_sub_i64(rh, t1, t2);
2599 tcg_gen_mov_i64(rl, t0);
2600 tcg_temp_free_i64(t0);
2601 tcg_temp_free_i64(t1);
2602 tcg_temp_free_i64(t2);
2603 }
2604
2605 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2606 {
2607 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2608 }
2609
2610 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2611 {
2612 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2613 }
2614
2615 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2616 {
2617 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2618 }
2619
2620 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2621 {
2622 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2623 }
2624
2625 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2626 {
2627 TCGv_i64 t = tcg_temp_new_i64();
2628
2629 tcg_gen_sari_i64(t, a, 63);
2630 tcg_gen_xor_i64(ret, a, t);
2631 tcg_gen_sub_i64(ret, ret, t);
2632 tcg_temp_free_i64(t);
2633 }
2634
2635 /* Size changing operations. */
2636
2637 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2638 {
2639 if (TCG_TARGET_REG_BITS == 32) {
2640 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2641 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2642 tcg_gen_op2(INDEX_op_extrl_i64_i32,
2643 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2644 } else {
2645 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2646 }
2647 }
2648
2649 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2650 {
2651 if (TCG_TARGET_REG_BITS == 32) {
2652 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2653 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2654 tcg_gen_op2(INDEX_op_extrh_i64_i32,
2655 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2656 } else {
2657 TCGv_i64 t = tcg_temp_new_i64();
2658 tcg_gen_shri_i64(t, arg, 32);
2659 tcg_gen_mov_i32(ret, (TCGv_i32)t);
2660 tcg_temp_free_i64(t);
2661 }
2662 }
2663
2664 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2665 {
2666 if (TCG_TARGET_REG_BITS == 32) {
2667 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2668 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2669 } else {
2670 tcg_gen_op2(INDEX_op_extu_i32_i64,
2671 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2672 }
2673 }
2674
2675 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2676 {
2677 if (TCG_TARGET_REG_BITS == 32) {
2678 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2679 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2680 } else {
2681 tcg_gen_op2(INDEX_op_ext_i32_i64,
2682 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2683 }
2684 }
2685
2686 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2687 {
2688 TCGv_i64 tmp;
2689
2690 if (TCG_TARGET_REG_BITS == 32) {
2691 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2692 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2693 return;
2694 }
2695
2696 tmp = tcg_temp_new_i64();
2697 /* These extensions are only needed for type correctness.
2698 We may be able to do better given target specific information. */
2699 tcg_gen_extu_i32_i64(tmp, high);
2700 tcg_gen_extu_i32_i64(dest, low);
2701 /* If deposit is available, use it. Otherwise use the extra
2702 knowledge that we have of the zero-extensions above. */
2703 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2704 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2705 } else {
2706 tcg_gen_shli_i64(tmp, tmp, 32);
2707 tcg_gen_or_i64(dest, dest, tmp);
2708 }
2709 tcg_temp_free_i64(tmp);
2710 }
2711
2712 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2713 {
2714 if (TCG_TARGET_REG_BITS == 32) {
2715 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2716 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2717 } else {
2718 tcg_gen_extrl_i64_i32(lo, arg);
2719 tcg_gen_extrh_i64_i32(hi, arg);
2720 }
2721 }
2722
2723 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2724 {
2725 tcg_gen_ext32u_i64(lo, arg);
2726 tcg_gen_shri_i64(hi, arg, 32);
2727 }
2728
2729 /* QEMU specific operations. */
2730
2731 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
2732 {
2733 /*
2734 * Let the jit code return the read-only version of the
2735 * TranslationBlock, so that we minimize the pc-relative
2736 * distance of the address of the exit_tb code to TB.
2737 * This will improve utilization of pc-relative address loads.
2738 *
2739 * TODO: Move this to translator_loop, so that all const
2740 * TranslationBlock pointers refer to read-only memory.
2741 * This requires coordination with targets that do not use
2742 * the translator_loop.
2743 */
2744 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
2745
2746 if (tb == NULL) {
2747 tcg_debug_assert(idx == 0);
2748 } else if (idx <= TB_EXIT_IDXMAX) {
2749 #ifdef CONFIG_DEBUG_TCG
2750 /* This is an exit following a goto_tb. Verify that we have
2751 seen this numbered exit before, via tcg_gen_goto_tb. */
2752 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2753 #endif
2754 } else {
2755 /* This is an exit via the exitreq label. */
2756 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2757 }
2758
2759 plugin_gen_disable_mem_helpers();
2760 tcg_gen_op1i(INDEX_op_exit_tb, val);
2761 }
2762
2763 void tcg_gen_goto_tb(unsigned idx)
2764 {
2765 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
2766 tcg_debug_assert(!(tcg_ctx->tb_cflags & CF_NO_GOTO_TB));
2767 /* We only support two chained exits. */
2768 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2769 #ifdef CONFIG_DEBUG_TCG
2770 /* Verify that we haven't seen this numbered exit before. */
2771 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2772 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2773 #endif
2774 plugin_gen_disable_mem_helpers();
2775 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2776 }
2777
2778 void tcg_gen_lookup_and_goto_ptr(void)
2779 {
2780 TCGv_ptr ptr;
2781
2782 if (tcg_ctx->tb_cflags & CF_NO_GOTO_PTR) {
2783 tcg_gen_exit_tb(NULL, 0);
2784 return;
2785 }
2786
2787 plugin_gen_disable_mem_helpers();
2788 ptr = tcg_temp_new_ptr();
2789 gen_helper_lookup_tb_ptr(ptr, cpu_env);
2790 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2791 tcg_temp_free_ptr(ptr);
2792 }
2793
2794 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
2795 {
2796 /* Trigger the asserts within as early as possible. */
2797 unsigned a_bits = get_alignment_bits(op);
2798
2799 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
2800 if (a_bits == (op & MO_SIZE)) {
2801 op = (op & ~MO_AMASK) | MO_ALIGN;
2802 }
2803
2804 switch (op & MO_SIZE) {
2805 case MO_8:
2806 op &= ~MO_BSWAP;
2807 break;
2808 case MO_16:
2809 break;
2810 case MO_32:
2811 if (!is64) {
2812 op &= ~MO_SIGN;
2813 }
2814 break;
2815 case MO_64:
2816 if (is64) {
2817 op &= ~MO_SIGN;
2818 break;
2819 }
2820 /* fall through */
2821 default:
2822 g_assert_not_reached();
2823 }
2824 if (st) {
2825 op &= ~MO_SIGN;
2826 }
2827 return op;
2828 }
2829
2830 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2831 MemOp memop, TCGArg idx)
2832 {
2833 MemOpIdx oi = make_memop_idx(memop, idx);
2834 #if TARGET_LONG_BITS == 32
2835 tcg_gen_op3i_i32(opc, val, addr, oi);
2836 #else
2837 if (TCG_TARGET_REG_BITS == 32) {
2838 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2839 } else {
2840 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2841 }
2842 #endif
2843 }
2844
2845 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2846 MemOp memop, TCGArg idx)
2847 {
2848 MemOpIdx oi = make_memop_idx(memop, idx);
2849 #if TARGET_LONG_BITS == 32
2850 if (TCG_TARGET_REG_BITS == 32) {
2851 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2852 } else {
2853 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2854 }
2855 #else
2856 if (TCG_TARGET_REG_BITS == 32) {
2857 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2858 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2859 } else {
2860 tcg_gen_op3i_i64(opc, val, addr, oi);
2861 }
2862 #endif
2863 }
2864
2865 static void tcg_gen_req_mo(TCGBar type)
2866 {
2867 #ifdef TCG_GUEST_DEFAULT_MO
2868 type &= TCG_GUEST_DEFAULT_MO;
2869 #endif
2870 type &= ~TCG_TARGET_DEFAULT_MO;
2871 if (type) {
2872 tcg_gen_mb(type | TCG_BAR_SC);
2873 }
2874 }
2875
2876 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr)
2877 {
2878 #ifdef CONFIG_PLUGIN
2879 if (tcg_ctx->plugin_insn != NULL) {
2880 /* Save a copy of the vaddr for use after a load. */
2881 TCGv temp = tcg_temp_new();
2882 tcg_gen_mov_tl(temp, vaddr);
2883 return temp;
2884 }
2885 #endif
2886 return vaddr;
2887 }
2888
2889 static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi,
2890 enum qemu_plugin_mem_rw rw)
2891 {
2892 #ifdef CONFIG_PLUGIN
2893 if (tcg_ctx->plugin_insn != NULL) {
2894 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
2895 plugin_gen_empty_mem_callback(vaddr, info);
2896 tcg_temp_free(vaddr);
2897 }
2898 #endif
2899 }
2900
2901 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2902 {
2903 MemOp orig_memop;
2904 MemOpIdx oi;
2905
2906 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2907 memop = tcg_canonicalize_memop(memop, 0, 0);
2908 oi = make_memop_idx(memop, idx);
2909
2910 orig_memop = memop;
2911 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2912 memop &= ~MO_BSWAP;
2913 /* The bswap primitive benefits from zero-extended input. */
2914 if ((memop & MO_SSIZE) == MO_SW) {
2915 memop &= ~MO_SIGN;
2916 }
2917 }
2918
2919 addr = plugin_prep_mem_callbacks(addr);
2920 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2921 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
2922
2923 if ((orig_memop ^ memop) & MO_BSWAP) {
2924 switch (orig_memop & MO_SIZE) {
2925 case MO_16:
2926 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
2927 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2928 : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
2929 break;
2930 case MO_32:
2931 tcg_gen_bswap32_i32(val, val);
2932 break;
2933 default:
2934 g_assert_not_reached();
2935 }
2936 }
2937 }
2938
2939 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2940 {
2941 TCGv_i32 swap = NULL;
2942 MemOpIdx oi;
2943
2944 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2945 memop = tcg_canonicalize_memop(memop, 0, 1);
2946 oi = make_memop_idx(memop, idx);
2947
2948 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2949 swap = tcg_temp_new_i32();
2950 switch (memop & MO_SIZE) {
2951 case MO_16:
2952 tcg_gen_bswap16_i32(swap, val, 0);
2953 break;
2954 case MO_32:
2955 tcg_gen_bswap32_i32(swap, val);
2956 break;
2957 default:
2958 g_assert_not_reached();
2959 }
2960 val = swap;
2961 memop &= ~MO_BSWAP;
2962 }
2963
2964 addr = plugin_prep_mem_callbacks(addr);
2965 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
2966 gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
2967 } else {
2968 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2969 }
2970 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
2971
2972 if (swap) {
2973 tcg_temp_free_i32(swap);
2974 }
2975 }
2976
2977 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2978 {
2979 MemOp orig_memop;
2980 MemOpIdx oi;
2981
2982 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2983 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2984 if (memop & MO_SIGN) {
2985 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2986 } else {
2987 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2988 }
2989 return;
2990 }
2991
2992 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2993 memop = tcg_canonicalize_memop(memop, 1, 0);
2994 oi = make_memop_idx(memop, idx);
2995
2996 orig_memop = memop;
2997 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2998 memop &= ~MO_BSWAP;
2999 /* The bswap primitive benefits from zero-extended input. */
3000 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
3001 memop &= ~MO_SIGN;
3002 }
3003 }
3004
3005 addr = plugin_prep_mem_callbacks(addr);
3006 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
3007 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
3008
3009 if ((orig_memop ^ memop) & MO_BSWAP) {
3010 int flags = (orig_memop & MO_SIGN
3011 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
3012 : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
3013 switch (orig_memop & MO_SIZE) {
3014 case MO_16:
3015 tcg_gen_bswap16_i64(val, val, flags);
3016 break;
3017 case MO_32:
3018 tcg_gen_bswap32_i64(val, val, flags);
3019 break;
3020 case MO_64:
3021 tcg_gen_bswap64_i64(val, val);
3022 break;
3023 default:
3024 g_assert_not_reached();
3025 }
3026 }
3027 }
3028
3029 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
3030 {
3031 TCGv_i64 swap = NULL;
3032 MemOpIdx oi;
3033
3034 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
3035 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
3036 return;
3037 }
3038
3039 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
3040 memop = tcg_canonicalize_memop(memop, 1, 1);
3041 oi = make_memop_idx(memop, idx);
3042
3043 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3044 swap = tcg_temp_new_i64();
3045 switch (memop & MO_SIZE) {
3046 case MO_16:
3047 tcg_gen_bswap16_i64(swap, val, 0);
3048 break;
3049 case MO_32:
3050 tcg_gen_bswap32_i64(swap, val, 0);
3051 break;
3052 case MO_64:
3053 tcg_gen_bswap64_i64(swap, val);
3054 break;
3055 default:
3056 g_assert_not_reached();
3057 }
3058 val = swap;
3059 memop &= ~MO_BSWAP;
3060 }
3061
3062 addr = plugin_prep_mem_callbacks(addr);
3063 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
3064 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
3065
3066 if (swap) {
3067 tcg_temp_free_i64(swap);
3068 }
3069 }
3070
3071 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
3072 {
3073 switch (opc & MO_SSIZE) {
3074 case MO_SB:
3075 tcg_gen_ext8s_i32(ret, val);
3076 break;
3077 case MO_UB:
3078 tcg_gen_ext8u_i32(ret, val);
3079 break;
3080 case MO_SW:
3081 tcg_gen_ext16s_i32(ret, val);
3082 break;
3083 case MO_UW:
3084 tcg_gen_ext16u_i32(ret, val);
3085 break;
3086 default:
3087 tcg_gen_mov_i32(ret, val);
3088 break;
3089 }
3090 }
3091
3092 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
3093 {
3094 switch (opc & MO_SSIZE) {
3095 case MO_SB:
3096 tcg_gen_ext8s_i64(ret, val);
3097 break;
3098 case MO_UB:
3099 tcg_gen_ext8u_i64(ret, val);
3100 break;
3101 case MO_SW:
3102 tcg_gen_ext16s_i64(ret, val);
3103 break;
3104 case MO_UW:
3105 tcg_gen_ext16u_i64(ret, val);
3106 break;
3107 case MO_SL:
3108 tcg_gen_ext32s_i64(ret, val);
3109 break;
3110 case MO_UL:
3111 tcg_gen_ext32u_i64(ret, val);
3112 break;
3113 default:
3114 tcg_gen_mov_i64(ret, val);
3115 break;
3116 }
3117 }
3118
3119 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3120 TCGv_i32, TCGv_i32, TCGv_i32);
3121 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3122 TCGv_i64, TCGv_i64, TCGv_i32);
3123 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3124 TCGv_i32, TCGv_i32);
3125 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3126 TCGv_i64, TCGv_i32);
3127
3128 #ifdef CONFIG_ATOMIC64
3129 # define WITH_ATOMIC64(X) X,
3130 #else
3131 # define WITH_ATOMIC64(X)
3132 #endif
3133
3134 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
3135 [MO_8] = gen_helper_atomic_cmpxchgb,
3136 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3137 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3138 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3139 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
3140 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3141 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
3142 };
3143
3144 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3145 TCGv_i32 newv, TCGArg idx, MemOp memop)
3146 {
3147 memop = tcg_canonicalize_memop(memop, 0, 0);
3148
3149 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3150 TCGv_i32 t1 = tcg_temp_new_i32();
3151 TCGv_i32 t2 = tcg_temp_new_i32();
3152
3153 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3154
3155 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3156 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3157 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3158 tcg_temp_free_i32(t2);
3159
3160 if (memop & MO_SIGN) {
3161 tcg_gen_ext_i32(retv, t1, memop);
3162 } else {
3163 tcg_gen_mov_i32(retv, t1);
3164 }
3165 tcg_temp_free_i32(t1);
3166 } else {
3167 gen_atomic_cx_i32 gen;
3168 MemOpIdx oi;
3169
3170 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3171 tcg_debug_assert(gen != NULL);
3172
3173 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3174 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3175
3176 if (memop & MO_SIGN) {
3177 tcg_gen_ext_i32(retv, retv, memop);
3178 }
3179 }
3180 }
3181
3182 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3183 TCGv_i64 newv, TCGArg idx, MemOp memop)
3184 {
3185 memop = tcg_canonicalize_memop(memop, 1, 0);
3186
3187 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3188 TCGv_i64 t1 = tcg_temp_new_i64();
3189 TCGv_i64 t2 = tcg_temp_new_i64();
3190
3191 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
3192
3193 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3194 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3195 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3196 tcg_temp_free_i64(t2);
3197
3198 if (memop & MO_SIGN) {
3199 tcg_gen_ext_i64(retv, t1, memop);
3200 } else {
3201 tcg_gen_mov_i64(retv, t1);
3202 }
3203 tcg_temp_free_i64(t1);
3204 } else if ((memop & MO_SIZE) == MO_64) {
3205 #ifdef CONFIG_ATOMIC64
3206 gen_atomic_cx_i64 gen;
3207 MemOpIdx oi;
3208
3209 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3210 tcg_debug_assert(gen != NULL);
3211
3212 oi = make_memop_idx(memop, idx);
3213 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3214 #else
3215 gen_helper_exit_atomic(cpu_env);
3216 /* Produce a result, so that we have a well-formed opcode stream
3217 with respect to uses of the result in the (dead) code following. */
3218 tcg_gen_movi_i64(retv, 0);
3219 #endif /* CONFIG_ATOMIC64 */
3220 } else {
3221 TCGv_i32 c32 = tcg_temp_new_i32();
3222 TCGv_i32 n32 = tcg_temp_new_i32();
3223 TCGv_i32 r32 = tcg_temp_new_i32();
3224
3225 tcg_gen_extrl_i64_i32(c32, cmpv);
3226 tcg_gen_extrl_i64_i32(n32, newv);
3227 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3228 tcg_temp_free_i32(c32);
3229 tcg_temp_free_i32(n32);
3230
3231 tcg_gen_extu_i32_i64(retv, r32);
3232 tcg_temp_free_i32(r32);
3233
3234 if (memop & MO_SIGN) {
3235 tcg_gen_ext_i64(retv, retv, memop);
3236 }
3237 }
3238 }
3239
3240 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3241 TCGArg idx, MemOp memop, bool new_val,
3242 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3243 {
3244 TCGv_i32 t1 = tcg_temp_new_i32();
3245 TCGv_i32 t2 = tcg_temp_new_i32();
3246
3247 memop = tcg_canonicalize_memop(memop, 0, 0);
3248
3249 tcg_gen_qemu_ld_i32(t1, addr, idx, memop);
3250 tcg_gen_ext_i32(t2, val, memop);
3251 gen(t2, t1, t2);
3252 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3253
3254 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3255 tcg_temp_free_i32(t1);
3256 tcg_temp_free_i32(t2);
3257 }
3258
3259 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3260 TCGArg idx, MemOp memop, void * const table[])
3261 {
3262 gen_atomic_op_i32 gen;
3263 MemOpIdx oi;
3264
3265 memop = tcg_canonicalize_memop(memop, 0, 0);
3266
3267 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3268 tcg_debug_assert(gen != NULL);
3269
3270 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3271 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3272
3273 if (memop & MO_SIGN) {
3274 tcg_gen_ext_i32(ret, ret, memop);
3275 }
3276 }
3277
3278 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3279 TCGArg idx, MemOp memop, bool new_val,
3280 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3281 {
3282 TCGv_i64 t1 = tcg_temp_new_i64();
3283 TCGv_i64 t2 = tcg_temp_new_i64();
3284
3285 memop = tcg_canonicalize_memop(memop, 1, 0);
3286
3287 tcg_gen_qemu_ld_i64(t1, addr, idx, memop);
3288 tcg_gen_ext_i64(t2, val, memop);
3289 gen(t2, t1, t2);
3290 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3291
3292 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3293 tcg_temp_free_i64(t1);
3294 tcg_temp_free_i64(t2);
3295 }
3296
3297 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3298 TCGArg idx, MemOp memop, void * const table[])
3299 {
3300 memop = tcg_canonicalize_memop(memop, 1, 0);
3301
3302 if ((memop & MO_SIZE) == MO_64) {
3303 #ifdef CONFIG_ATOMIC64
3304 gen_atomic_op_i64 gen;
3305 MemOpIdx oi;
3306
3307 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3308 tcg_debug_assert(gen != NULL);
3309
3310 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3311 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3312 #else
3313 gen_helper_exit_atomic(cpu_env);
3314 /* Produce a result, so that we have a well-formed opcode stream
3315 with respect to uses of the result in the (dead) code following. */
3316 tcg_gen_movi_i64(ret, 0);
3317 #endif /* CONFIG_ATOMIC64 */
3318 } else {
3319 TCGv_i32 v32 = tcg_temp_new_i32();
3320 TCGv_i32 r32 = tcg_temp_new_i32();
3321
3322 tcg_gen_extrl_i64_i32(v32, val);
3323 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3324 tcg_temp_free_i32(v32);
3325
3326 tcg_gen_extu_i32_i64(ret, r32);
3327 tcg_temp_free_i32(r32);
3328
3329 if (memop & MO_SIGN) {
3330 tcg_gen_ext_i64(ret, ret, memop);
3331 }
3332 }
3333 }
3334
3335 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3336 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
3337 [MO_8] = gen_helper_atomic_##NAME##b, \
3338 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3339 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3340 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3341 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
3342 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3343 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
3344 }; \
3345 void tcg_gen_atomic_##NAME##_i32 \
3346 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \
3347 { \
3348 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3349 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3350 } else { \
3351 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3352 tcg_gen_##OP##_i32); \
3353 } \
3354 } \
3355 void tcg_gen_atomic_##NAME##_i64 \
3356 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \
3357 { \
3358 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \
3359 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3360 } else { \
3361 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3362 tcg_gen_##OP##_i64); \
3363 } \
3364 }
3365
3366 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3367 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3368 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3369 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3370 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3371 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3372 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3373 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3374
3375 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3376 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3377 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3378 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3379 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3380 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3381 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3382 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3383
3384 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3385 {
3386 tcg_gen_mov_i32(r, b);
3387 }
3388
3389 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3390 {
3391 tcg_gen_mov_i64(r, b);
3392 }
3393
3394 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3395
3396 #undef GEN_ATOMIC_HELPER
QEmu