?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hw/pci-host: Use register definitions from PCI standard
[qemu.git] / tcg / tcg-op.c
blob9fa9f1b0fd2472c9ba05c1b8fdd55e353db24624
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "exec/exec-all.h"
27 #include "tcg/tcg.h"
28 #include "tcg/tcg-op.h"
29 #include "tcg/tcg-mo.h"
30 #include "exec/plugin-gen.h"
31 #include "tcg-internal.h"
32
33
34 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
35 {
36 TCGOp *op = tcg_emit_op(opc, 1);
37 op->args[0] = a1;
38 }
39
40 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
41 {
42 TCGOp *op = tcg_emit_op(opc, 2);
43 op->args[0] = a1;
44 op->args[1] = a2;
45 }
46
47 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
48 {
49 TCGOp *op = tcg_emit_op(opc, 3);
50 op->args[0] = a1;
51 op->args[1] = a2;
52 op->args[2] = a3;
53 }
54
55 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
56 {
57 TCGOp *op = tcg_emit_op(opc, 4);
58 op->args[0] = a1;
59 op->args[1] = a2;
60 op->args[2] = a3;
61 op->args[3] = a4;
62 }
63
64 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
65 TCGArg a4, TCGArg a5)
66 {
67 TCGOp *op = tcg_emit_op(opc, 5);
68 op->args[0] = a1;
69 op->args[1] = a2;
70 op->args[2] = a3;
71 op->args[3] = a4;
72 op->args[4] = a5;
73 }
74
75 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
76 TCGArg a4, TCGArg a5, TCGArg a6)
77 {
78 TCGOp *op = tcg_emit_op(opc, 6);
79 op->args[0] = a1;
80 op->args[1] = a2;
81 op->args[2] = a3;
82 op->args[3] = a4;
83 op->args[4] = a5;
84 op->args[5] = a6;
85 }
86
87 void tcg_gen_mb(TCGBar mb_type)
88 {
89 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) {
90 tcg_gen_op1(INDEX_op_mb, mb_type);
91 }
92 }
93
94 /* 32 bit ops */
95
96 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
97 {
98 tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
99 }
100
101 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
102 {
103 /* some cases can be optimized here */
104 if (arg2 == 0) {
105 tcg_gen_mov_i32(ret, arg1);
106 } else {
107 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
108 }
109 }
110
111 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
112 {
113 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
114 /* Don't recurse with tcg_gen_neg_i32. */
115 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
116 } else {
117 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
118 }
119 }
120
121 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
122 {
123 /* some cases can be optimized here */
124 if (arg2 == 0) {
125 tcg_gen_mov_i32(ret, arg1);
126 } else {
127 tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2));
128 }
129 }
130
131 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
132 {
133 /* Some cases can be optimized here. */
134 switch (arg2) {
135 case 0:
136 tcg_gen_movi_i32(ret, 0);
137 return;
138 case -1:
139 tcg_gen_mov_i32(ret, arg1);
140 return;
141 case 0xff:
142 /* Don't recurse with tcg_gen_ext8u_i32. */
143 if (TCG_TARGET_HAS_ext8u_i32) {
144 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
145 return;
146 }
147 break;
148 case 0xffff:
149 if (TCG_TARGET_HAS_ext16u_i32) {
150 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
151 return;
152 }
153 break;
154 }
155
156 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
157 }
158
159 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
160 {
161 /* Some cases can be optimized here. */
162 if (arg2 == -1) {
163 tcg_gen_movi_i32(ret, -1);
164 } else if (arg2 == 0) {
165 tcg_gen_mov_i32(ret, arg1);
166 } else {
167 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
168 }
169 }
170
171 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
172 {
173 /* Some cases can be optimized here. */
174 if (arg2 == 0) {
175 tcg_gen_mov_i32(ret, arg1);
176 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
177 /* Don't recurse with tcg_gen_not_i32. */
178 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
179 } else {
180 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
181 }
182 }
183
184 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
185 {
186 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
187 if (arg2 == 0) {
188 tcg_gen_mov_i32(ret, arg1);
189 } else {
190 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
191 }
192 }
193
194 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
195 {
196 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
197 if (arg2 == 0) {
198 tcg_gen_mov_i32(ret, arg1);
199 } else {
200 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
201 }
202 }
203
204 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
205 {
206 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
207 if (arg2 == 0) {
208 tcg_gen_mov_i32(ret, arg1);
209 } else {
210 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
211 }
212 }
213
214 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
215 {
216 if (cond == TCG_COND_ALWAYS) {
217 tcg_gen_br(l);
218 } else if (cond != TCG_COND_NEVER) {
219 l->refs++;
220 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
221 }
222 }
223
224 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
225 {
226 if (cond == TCG_COND_ALWAYS) {
227 tcg_gen_br(l);
228 } else if (cond != TCG_COND_NEVER) {
229 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
230 }
231 }
232
233 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
234 TCGv_i32 arg1, TCGv_i32 arg2)
235 {
236 if (cond == TCG_COND_ALWAYS) {
237 tcg_gen_movi_i32(ret, 1);
238 } else if (cond == TCG_COND_NEVER) {
239 tcg_gen_movi_i32(ret, 0);
240 } else {
241 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
242 }
243 }
244
245 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
246 TCGv_i32 arg1, int32_t arg2)
247 {
248 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
249 }
250
251 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
252 {
253 if (arg2 == 0) {
254 tcg_gen_movi_i32(ret, 0);
255 } else if (is_power_of_2(arg2)) {
256 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
257 } else {
258 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
259 }
260 }
261
262 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
263 {
264 if (TCG_TARGET_HAS_div_i32) {
265 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
266 } else if (TCG_TARGET_HAS_div2_i32) {
267 TCGv_i32 t0 = tcg_temp_new_i32();
268 tcg_gen_sari_i32(t0, arg1, 31);
269 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
270 tcg_temp_free_i32(t0);
271 } else {
272 gen_helper_div_i32(ret, arg1, arg2);
273 }
274 }
275
276 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
277 {
278 if (TCG_TARGET_HAS_rem_i32) {
279 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
280 } else if (TCG_TARGET_HAS_div_i32) {
281 TCGv_i32 t0 = tcg_temp_new_i32();
282 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
283 tcg_gen_mul_i32(t0, t0, arg2);
284 tcg_gen_sub_i32(ret, arg1, t0);
285 tcg_temp_free_i32(t0);
286 } else if (TCG_TARGET_HAS_div2_i32) {
287 TCGv_i32 t0 = tcg_temp_new_i32();
288 tcg_gen_sari_i32(t0, arg1, 31);
289 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
290 tcg_temp_free_i32(t0);
291 } else {
292 gen_helper_rem_i32(ret, arg1, arg2);
293 }
294 }
295
296 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
297 {
298 if (TCG_TARGET_HAS_div_i32) {
299 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
300 } else if (TCG_TARGET_HAS_div2_i32) {
301 TCGv_i32 t0 = tcg_temp_new_i32();
302 tcg_gen_movi_i32(t0, 0);
303 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
304 tcg_temp_free_i32(t0);
305 } else {
306 gen_helper_divu_i32(ret, arg1, arg2);
307 }
308 }
309
310 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
311 {
312 if (TCG_TARGET_HAS_rem_i32) {
313 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
314 } else if (TCG_TARGET_HAS_div_i32) {
315 TCGv_i32 t0 = tcg_temp_new_i32();
316 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
317 tcg_gen_mul_i32(t0, t0, arg2);
318 tcg_gen_sub_i32(ret, arg1, t0);
319 tcg_temp_free_i32(t0);
320 } else if (TCG_TARGET_HAS_div2_i32) {
321 TCGv_i32 t0 = tcg_temp_new_i32();
322 tcg_gen_movi_i32(t0, 0);
323 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
324 tcg_temp_free_i32(t0);
325 } else {
326 gen_helper_remu_i32(ret, arg1, arg2);
327 }
328 }
329
330 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
331 {
332 if (TCG_TARGET_HAS_andc_i32) {
333 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
334 } else {
335 TCGv_i32 t0 = tcg_temp_new_i32();
336 tcg_gen_not_i32(t0, arg2);
337 tcg_gen_and_i32(ret, arg1, t0);
338 tcg_temp_free_i32(t0);
339 }
340 }
341
342 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
343 {
344 if (TCG_TARGET_HAS_eqv_i32) {
345 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
346 } else {
347 tcg_gen_xor_i32(ret, arg1, arg2);
348 tcg_gen_not_i32(ret, ret);
349 }
350 }
351
352 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
353 {
354 if (TCG_TARGET_HAS_nand_i32) {
355 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
356 } else {
357 tcg_gen_and_i32(ret, arg1, arg2);
358 tcg_gen_not_i32(ret, ret);
359 }
360 }
361
362 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
363 {
364 if (TCG_TARGET_HAS_nor_i32) {
365 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
366 } else {
367 tcg_gen_or_i32(ret, arg1, arg2);
368 tcg_gen_not_i32(ret, ret);
369 }
370 }
371
372 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
373 {
374 if (TCG_TARGET_HAS_orc_i32) {
375 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
376 } else {
377 TCGv_i32 t0 = tcg_temp_new_i32();
378 tcg_gen_not_i32(t0, arg2);
379 tcg_gen_or_i32(ret, arg1, t0);
380 tcg_temp_free_i32(t0);
381 }
382 }
383
384 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
385 {
386 if (TCG_TARGET_HAS_clz_i32) {
387 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
388 } else if (TCG_TARGET_HAS_clz_i64) {
389 TCGv_i64 t1 = tcg_temp_new_i64();
390 TCGv_i64 t2 = tcg_temp_new_i64();
391 tcg_gen_extu_i32_i64(t1, arg1);
392 tcg_gen_extu_i32_i64(t2, arg2);
393 tcg_gen_addi_i64(t2, t2, 32);
394 tcg_gen_clz_i64(t1, t1, t2);
395 tcg_gen_extrl_i64_i32(ret, t1);
396 tcg_temp_free_i64(t1);
397 tcg_temp_free_i64(t2);
398 tcg_gen_subi_i32(ret, ret, 32);
399 } else {
400 gen_helper_clz_i32(ret, arg1, arg2);
401 }
402 }
403
404 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
405 {
406 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
407 }
408
409 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
410 {
411 if (TCG_TARGET_HAS_ctz_i32) {
412 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
413 } else if (TCG_TARGET_HAS_ctz_i64) {
414 TCGv_i64 t1 = tcg_temp_new_i64();
415 TCGv_i64 t2 = tcg_temp_new_i64();
416 tcg_gen_extu_i32_i64(t1, arg1);
417 tcg_gen_extu_i32_i64(t2, arg2);
418 tcg_gen_ctz_i64(t1, t1, t2);
419 tcg_gen_extrl_i64_i32(ret, t1);
420 tcg_temp_free_i64(t1);
421 tcg_temp_free_i64(t2);
422 } else if (TCG_TARGET_HAS_ctpop_i32
423 || TCG_TARGET_HAS_ctpop_i64
424 || TCG_TARGET_HAS_clz_i32
425 || TCG_TARGET_HAS_clz_i64) {
426 TCGv_i32 z, t = tcg_temp_new_i32();
427
428 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
429 tcg_gen_subi_i32(t, arg1, 1);
430 tcg_gen_andc_i32(t, t, arg1);
431 tcg_gen_ctpop_i32(t, t);
432 } else {
433 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
434 tcg_gen_neg_i32(t, arg1);
435 tcg_gen_and_i32(t, t, arg1);
436 tcg_gen_clzi_i32(t, t, 32);
437 tcg_gen_xori_i32(t, t, 31);
438 }
439 z = tcg_constant_i32(0);
440 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
441 tcg_temp_free_i32(t);
442 } else {
443 gen_helper_ctz_i32(ret, arg1, arg2);
444 }
445 }
446
447 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
448 {
449 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
450 /* This equivalence has the advantage of not requiring a fixup. */
451 TCGv_i32 t = tcg_temp_new_i32();
452 tcg_gen_subi_i32(t, arg1, 1);
453 tcg_gen_andc_i32(t, t, arg1);
454 tcg_gen_ctpop_i32(ret, t);
455 tcg_temp_free_i32(t);
456 } else {
457 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
458 }
459 }
460
461 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
462 {
463 if (TCG_TARGET_HAS_clz_i32) {
464 TCGv_i32 t = tcg_temp_new_i32();
465 tcg_gen_sari_i32(t, arg, 31);
466 tcg_gen_xor_i32(t, t, arg);
467 tcg_gen_clzi_i32(t, t, 32);
468 tcg_gen_subi_i32(ret, t, 1);
469 tcg_temp_free_i32(t);
470 } else {
471 gen_helper_clrsb_i32(ret, arg);
472 }
473 }
474
475 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
476 {
477 if (TCG_TARGET_HAS_ctpop_i32) {
478 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
479 } else if (TCG_TARGET_HAS_ctpop_i64) {
480 TCGv_i64 t = tcg_temp_new_i64();
481 tcg_gen_extu_i32_i64(t, arg1);
482 tcg_gen_ctpop_i64(t, t);
483 tcg_gen_extrl_i64_i32(ret, t);
484 tcg_temp_free_i64(t);
485 } else {
486 gen_helper_ctpop_i32(ret, arg1);
487 }
488 }
489
490 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
491 {
492 if (TCG_TARGET_HAS_rot_i32) {
493 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
494 } else {
495 TCGv_i32 t0, t1;
496
497 t0 = tcg_temp_new_i32();
498 t1 = tcg_temp_new_i32();
499 tcg_gen_shl_i32(t0, arg1, arg2);
500 tcg_gen_subfi_i32(t1, 32, arg2);
501 tcg_gen_shr_i32(t1, arg1, t1);
502 tcg_gen_or_i32(ret, t0, t1);
503 tcg_temp_free_i32(t0);
504 tcg_temp_free_i32(t1);
505 }
506 }
507
508 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
509 {
510 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
511 /* some cases can be optimized here */
512 if (arg2 == 0) {
513 tcg_gen_mov_i32(ret, arg1);
514 } else if (TCG_TARGET_HAS_rot_i32) {
515 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
516 } else {
517 TCGv_i32 t0, t1;
518 t0 = tcg_temp_new_i32();
519 t1 = tcg_temp_new_i32();
520 tcg_gen_shli_i32(t0, arg1, arg2);
521 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
522 tcg_gen_or_i32(ret, t0, t1);
523 tcg_temp_free_i32(t0);
524 tcg_temp_free_i32(t1);
525 }
526 }
527
528 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
529 {
530 if (TCG_TARGET_HAS_rot_i32) {
531 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
532 } else {
533 TCGv_i32 t0, t1;
534
535 t0 = tcg_temp_new_i32();
536 t1 = tcg_temp_new_i32();
537 tcg_gen_shr_i32(t0, arg1, arg2);
538 tcg_gen_subfi_i32(t1, 32, arg2);
539 tcg_gen_shl_i32(t1, arg1, t1);
540 tcg_gen_or_i32(ret, t0, t1);
541 tcg_temp_free_i32(t0);
542 tcg_temp_free_i32(t1);
543 }
544 }
545
546 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
547 {
548 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
549 /* some cases can be optimized here */
550 if (arg2 == 0) {
551 tcg_gen_mov_i32(ret, arg1);
552 } else {
553 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
554 }
555 }
556
557 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
558 unsigned int ofs, unsigned int len)
559 {
560 uint32_t mask;
561 TCGv_i32 t1;
562
563 tcg_debug_assert(ofs < 32);
564 tcg_debug_assert(len > 0);
565 tcg_debug_assert(len <= 32);
566 tcg_debug_assert(ofs + len <= 32);
567
568 if (len == 32) {
569 tcg_gen_mov_i32(ret, arg2);
570 return;
571 }
572 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
573 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
574 return;
575 }
576
577 t1 = tcg_temp_new_i32();
578
579 if (TCG_TARGET_HAS_extract2_i32) {
580 if (ofs + len == 32) {
581 tcg_gen_shli_i32(t1, arg1, len);
582 tcg_gen_extract2_i32(ret, t1, arg2, len);
583 goto done;
584 }
585 if (ofs == 0) {
586 tcg_gen_extract2_i32(ret, arg1, arg2, len);
587 tcg_gen_rotli_i32(ret, ret, len);
588 goto done;
589 }
590 }
591
592 mask = (1u << len) - 1;
593 if (ofs + len < 32) {
594 tcg_gen_andi_i32(t1, arg2, mask);
595 tcg_gen_shli_i32(t1, t1, ofs);
596 } else {
597 tcg_gen_shli_i32(t1, arg2, ofs);
598 }
599 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
600 tcg_gen_or_i32(ret, ret, t1);
601 done:
602 tcg_temp_free_i32(t1);
603 }
604
605 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
606 unsigned int ofs, unsigned int len)
607 {
608 tcg_debug_assert(ofs < 32);
609 tcg_debug_assert(len > 0);
610 tcg_debug_assert(len <= 32);
611 tcg_debug_assert(ofs + len <= 32);
612
613 if (ofs + len == 32) {
614 tcg_gen_shli_i32(ret, arg, ofs);
615 } else if (ofs == 0) {
616 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
617 } else if (TCG_TARGET_HAS_deposit_i32
618 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
619 TCGv_i32 zero = tcg_constant_i32(0);
620 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
621 } else {
622 /* To help two-operand hosts we prefer to zero-extend first,
623 which allows ARG to stay live. */
624 switch (len) {
625 case 16:
626 if (TCG_TARGET_HAS_ext16u_i32) {
627 tcg_gen_ext16u_i32(ret, arg);
628 tcg_gen_shli_i32(ret, ret, ofs);
629 return;
630 }
631 break;
632 case 8:
633 if (TCG_TARGET_HAS_ext8u_i32) {
634 tcg_gen_ext8u_i32(ret, arg);
635 tcg_gen_shli_i32(ret, ret, ofs);
636 return;
637 }
638 break;
639 }
640 /* Otherwise prefer zero-extension over AND for code size. */
641 switch (ofs + len) {
642 case 16:
643 if (TCG_TARGET_HAS_ext16u_i32) {
644 tcg_gen_shli_i32(ret, arg, ofs);
645 tcg_gen_ext16u_i32(ret, ret);
646 return;
647 }
648 break;
649 case 8:
650 if (TCG_TARGET_HAS_ext8u_i32) {
651 tcg_gen_shli_i32(ret, arg, ofs);
652 tcg_gen_ext8u_i32(ret, ret);
653 return;
654 }
655 break;
656 }
657 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
658 tcg_gen_shli_i32(ret, ret, ofs);
659 }
660 }
661
662 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
663 unsigned int ofs, unsigned int len)
664 {
665 tcg_debug_assert(ofs < 32);
666 tcg_debug_assert(len > 0);
667 tcg_debug_assert(len <= 32);
668 tcg_debug_assert(ofs + len <= 32);
669
670 /* Canonicalize certain special cases, even if extract is supported. */
671 if (ofs + len == 32) {
672 tcg_gen_shri_i32(ret, arg, 32 - len);
673 return;
674 }
675 if (ofs == 0) {
676 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
677 return;
678 }
679
680 if (TCG_TARGET_HAS_extract_i32
681 && TCG_TARGET_extract_i32_valid(ofs, len)) {
682 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
683 return;
684 }
685
686 /* Assume that zero-extension, if available, is cheaper than a shift. */
687 switch (ofs + len) {
688 case 16:
689 if (TCG_TARGET_HAS_ext16u_i32) {
690 tcg_gen_ext16u_i32(ret, arg);
691 tcg_gen_shri_i32(ret, ret, ofs);
692 return;
693 }
694 break;
695 case 8:
696 if (TCG_TARGET_HAS_ext8u_i32) {
697 tcg_gen_ext8u_i32(ret, arg);
698 tcg_gen_shri_i32(ret, ret, ofs);
699 return;
700 }
701 break;
702 }
703
704 /* ??? Ideally we'd know what values are available for immediate AND.
705 Assume that 8 bits are available, plus the special case of 16,
706 so that we get ext8u, ext16u. */
707 switch (len) {
708 case 1 ... 8: case 16:
709 tcg_gen_shri_i32(ret, arg, ofs);
710 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
711 break;
712 default:
713 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
714 tcg_gen_shri_i32(ret, ret, 32 - len);
715 break;
716 }
717 }
718
719 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
720 unsigned int ofs, unsigned int len)
721 {
722 tcg_debug_assert(ofs < 32);
723 tcg_debug_assert(len > 0);
724 tcg_debug_assert(len <= 32);
725 tcg_debug_assert(ofs + len <= 32);
726
727 /* Canonicalize certain special cases, even if extract is supported. */
728 if (ofs + len == 32) {
729 tcg_gen_sari_i32(ret, arg, 32 - len);
730 return;
731 }
732 if (ofs == 0) {
733 switch (len) {
734 case 16:
735 tcg_gen_ext16s_i32(ret, arg);
736 return;
737 case 8:
738 tcg_gen_ext8s_i32(ret, arg);
739 return;
740 }
741 }
742
743 if (TCG_TARGET_HAS_sextract_i32
744 && TCG_TARGET_extract_i32_valid(ofs, len)) {
745 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
746 return;
747 }
748
749 /* Assume that sign-extension, if available, is cheaper than a shift. */
750 switch (ofs + len) {
751 case 16:
752 if (TCG_TARGET_HAS_ext16s_i32) {
753 tcg_gen_ext16s_i32(ret, arg);
754 tcg_gen_sari_i32(ret, ret, ofs);
755 return;
756 }
757 break;
758 case 8:
759 if (TCG_TARGET_HAS_ext8s_i32) {
760 tcg_gen_ext8s_i32(ret, arg);
761 tcg_gen_sari_i32(ret, ret, ofs);
762 return;
763 }
764 break;
765 }
766 switch (len) {
767 case 16:
768 if (TCG_TARGET_HAS_ext16s_i32) {
769 tcg_gen_shri_i32(ret, arg, ofs);
770 tcg_gen_ext16s_i32(ret, ret);
771 return;
772 }
773 break;
774 case 8:
775 if (TCG_TARGET_HAS_ext8s_i32) {
776 tcg_gen_shri_i32(ret, arg, ofs);
777 tcg_gen_ext8s_i32(ret, ret);
778 return;
779 }
780 break;
781 }
782
783 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
784 tcg_gen_sari_i32(ret, ret, 32 - len);
785 }
786
787 /*
788 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
789 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
790 */
791 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
792 unsigned int ofs)
793 {
794 tcg_debug_assert(ofs <= 32);
795 if (ofs == 0) {
796 tcg_gen_mov_i32(ret, al);
797 } else if (ofs == 32) {
798 tcg_gen_mov_i32(ret, ah);
799 } else if (al == ah) {
800 tcg_gen_rotri_i32(ret, al, ofs);
801 } else if (TCG_TARGET_HAS_extract2_i32) {
802 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
803 } else {
804 TCGv_i32 t0 = tcg_temp_new_i32();
805 tcg_gen_shri_i32(t0, al, ofs);
806 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
807 tcg_temp_free_i32(t0);
808 }
809 }
810
811 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
812 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
813 {
814 if (cond == TCG_COND_ALWAYS) {
815 tcg_gen_mov_i32(ret, v1);
816 } else if (cond == TCG_COND_NEVER) {
817 tcg_gen_mov_i32(ret, v2);
818 } else if (TCG_TARGET_HAS_movcond_i32) {
819 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
820 } else {
821 TCGv_i32 t0 = tcg_temp_new_i32();
822 TCGv_i32 t1 = tcg_temp_new_i32();
823 tcg_gen_setcond_i32(cond, t0, c1, c2);
824 tcg_gen_neg_i32(t0, t0);
825 tcg_gen_and_i32(t1, v1, t0);
826 tcg_gen_andc_i32(ret, v2, t0);
827 tcg_gen_or_i32(ret, ret, t1);
828 tcg_temp_free_i32(t0);
829 tcg_temp_free_i32(t1);
830 }
831 }
832
833 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
834 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
835 {
836 if (TCG_TARGET_HAS_add2_i32) {
837 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
838 } else {
839 TCGv_i64 t0 = tcg_temp_new_i64();
840 TCGv_i64 t1 = tcg_temp_new_i64();
841 tcg_gen_concat_i32_i64(t0, al, ah);
842 tcg_gen_concat_i32_i64(t1, bl, bh);
843 tcg_gen_add_i64(t0, t0, t1);
844 tcg_gen_extr_i64_i32(rl, rh, t0);
845 tcg_temp_free_i64(t0);
846 tcg_temp_free_i64(t1);
847 }
848 }
849
850 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
851 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
852 {
853 if (TCG_TARGET_HAS_sub2_i32) {
854 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
855 } else {
856 TCGv_i64 t0 = tcg_temp_new_i64();
857 TCGv_i64 t1 = tcg_temp_new_i64();
858 tcg_gen_concat_i32_i64(t0, al, ah);
859 tcg_gen_concat_i32_i64(t1, bl, bh);
860 tcg_gen_sub_i64(t0, t0, t1);
861 tcg_gen_extr_i64_i32(rl, rh, t0);
862 tcg_temp_free_i64(t0);
863 tcg_temp_free_i64(t1);
864 }
865 }
866
867 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
868 {
869 if (TCG_TARGET_HAS_mulu2_i32) {
870 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
871 } else if (TCG_TARGET_HAS_muluh_i32) {
872 TCGv_i32 t = tcg_temp_new_i32();
873 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
874 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
875 tcg_gen_mov_i32(rl, t);
876 tcg_temp_free_i32(t);
877 } else {
878 TCGv_i64 t0 = tcg_temp_new_i64();
879 TCGv_i64 t1 = tcg_temp_new_i64();
880 tcg_gen_extu_i32_i64(t0, arg1);
881 tcg_gen_extu_i32_i64(t1, arg2);
882 tcg_gen_mul_i64(t0, t0, t1);
883 tcg_gen_extr_i64_i32(rl, rh, t0);
884 tcg_temp_free_i64(t0);
885 tcg_temp_free_i64(t1);
886 }
887 }
888
889 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
890 {
891 if (TCG_TARGET_HAS_muls2_i32) {
892 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
893 } else if (TCG_TARGET_HAS_mulsh_i32) {
894 TCGv_i32 t = tcg_temp_new_i32();
895 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
896 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
897 tcg_gen_mov_i32(rl, t);
898 tcg_temp_free_i32(t);
899 } else if (TCG_TARGET_REG_BITS == 32) {
900 TCGv_i32 t0 = tcg_temp_new_i32();
901 TCGv_i32 t1 = tcg_temp_new_i32();
902 TCGv_i32 t2 = tcg_temp_new_i32();
903 TCGv_i32 t3 = tcg_temp_new_i32();
904 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
905 /* Adjust for negative inputs. */
906 tcg_gen_sari_i32(t2, arg1, 31);
907 tcg_gen_sari_i32(t3, arg2, 31);
908 tcg_gen_and_i32(t2, t2, arg2);
909 tcg_gen_and_i32(t3, t3, arg1);
910 tcg_gen_sub_i32(rh, t1, t2);
911 tcg_gen_sub_i32(rh, rh, t3);
912 tcg_gen_mov_i32(rl, t0);
913 tcg_temp_free_i32(t0);
914 tcg_temp_free_i32(t1);
915 tcg_temp_free_i32(t2);
916 tcg_temp_free_i32(t3);
917 } else {
918 TCGv_i64 t0 = tcg_temp_new_i64();
919 TCGv_i64 t1 = tcg_temp_new_i64();
920 tcg_gen_ext_i32_i64(t0, arg1);
921 tcg_gen_ext_i32_i64(t1, arg2);
922 tcg_gen_mul_i64(t0, t0, t1);
923 tcg_gen_extr_i64_i32(rl, rh, t0);
924 tcg_temp_free_i64(t0);
925 tcg_temp_free_i64(t1);
926 }
927 }
928
929 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
930 {
931 if (TCG_TARGET_REG_BITS == 32) {
932 TCGv_i32 t0 = tcg_temp_new_i32();
933 TCGv_i32 t1 = tcg_temp_new_i32();
934 TCGv_i32 t2 = tcg_temp_new_i32();
935 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
936 /* Adjust for negative input for the signed arg1. */
937 tcg_gen_sari_i32(t2, arg1, 31);
938 tcg_gen_and_i32(t2, t2, arg2);
939 tcg_gen_sub_i32(rh, t1, t2);
940 tcg_gen_mov_i32(rl, t0);
941 tcg_temp_free_i32(t0);
942 tcg_temp_free_i32(t1);
943 tcg_temp_free_i32(t2);
944 } else {
945 TCGv_i64 t0 = tcg_temp_new_i64();
946 TCGv_i64 t1 = tcg_temp_new_i64();
947 tcg_gen_ext_i32_i64(t0, arg1);
948 tcg_gen_extu_i32_i64(t1, arg2);
949 tcg_gen_mul_i64(t0, t0, t1);
950 tcg_gen_extr_i64_i32(rl, rh, t0);
951 tcg_temp_free_i64(t0);
952 tcg_temp_free_i64(t1);
953 }
954 }
955
956 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
957 {
958 if (TCG_TARGET_HAS_ext8s_i32) {
959 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
960 } else {
961 tcg_gen_shli_i32(ret, arg, 24);
962 tcg_gen_sari_i32(ret, ret, 24);
963 }
964 }
965
966 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
967 {
968 if (TCG_TARGET_HAS_ext16s_i32) {
969 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
970 } else {
971 tcg_gen_shli_i32(ret, arg, 16);
972 tcg_gen_sari_i32(ret, ret, 16);
973 }
974 }
975
976 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
977 {
978 if (TCG_TARGET_HAS_ext8u_i32) {
979 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
980 } else {
981 tcg_gen_andi_i32(ret, arg, 0xffu);
982 }
983 }
984
985 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
986 {
987 if (TCG_TARGET_HAS_ext16u_i32) {
988 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
989 } else {
990 tcg_gen_andi_i32(ret, arg, 0xffffu);
991 }
992 }
993
994 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
995 {
996 /* Only one extension flag may be present. */
997 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
998
999 if (TCG_TARGET_HAS_bswap16_i32) {
1000 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
1001 } else {
1002 TCGv_i32 t0 = tcg_temp_new_i32();
1003 TCGv_i32 t1 = tcg_temp_new_i32();
1004
1005 tcg_gen_shri_i32(t0, arg, 8);
1006 if (!(flags & TCG_BSWAP_IZ)) {
1007 tcg_gen_ext8u_i32(t0, t0);
1008 }
1009
1010 if (flags & TCG_BSWAP_OS) {
1011 tcg_gen_shli_i32(t1, arg, 24);
1012 tcg_gen_sari_i32(t1, t1, 16);
1013 } else if (flags & TCG_BSWAP_OZ) {
1014 tcg_gen_ext8u_i32(t1, arg);
1015 tcg_gen_shli_i32(t1, t1, 8);
1016 } else {
1017 tcg_gen_shli_i32(t1, arg, 8);
1018 }
1019
1020 tcg_gen_or_i32(ret, t0, t1);
1021 tcg_temp_free_i32(t0);
1022 tcg_temp_free_i32(t1);
1023 }
1024 }
1025
1026 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1027 {
1028 if (TCG_TARGET_HAS_bswap32_i32) {
1029 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
1030 } else {
1031 TCGv_i32 t0 = tcg_temp_new_i32();
1032 TCGv_i32 t1 = tcg_temp_new_i32();
1033 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
1034
1035 /* arg = abcd */
1036 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */
1037 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */
1038 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */
1039 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */
1040 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */
1041
1042 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */
1043 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */
1044 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */
1045
1046 tcg_temp_free_i32(t0);
1047 tcg_temp_free_i32(t1);
1048 }
1049 }
1050
1051 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg)
1052 {
1053 /* Swapping 2 16-bit elements is a rotate. */
1054 tcg_gen_rotli_i32(ret, arg, 16);
1055 }
1056
1057 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1058 {
1059 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1060 }
1061
1062 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1063 {
1064 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1065 }
1066
1067 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1068 {
1069 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1070 }
1071
1072 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1073 {
1074 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1075 }
1076
1077 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1078 {
1079 TCGv_i32 t = tcg_temp_new_i32();
1080
1081 tcg_gen_sari_i32(t, a, 31);
1082 tcg_gen_xor_i32(ret, a, t);
1083 tcg_gen_sub_i32(ret, ret, t);
1084 tcg_temp_free_i32(t);
1085 }
1086
1087 /* 64-bit ops */
1088
1089 #if TCG_TARGET_REG_BITS == 32
1090 /* These are all inline for TCG_TARGET_REG_BITS == 64. */
1091
1092 void tcg_gen_discard_i64(TCGv_i64 arg)
1093 {
1094 tcg_gen_discard_i32(TCGV_LOW(arg));
1095 tcg_gen_discard_i32(TCGV_HIGH(arg));
1096 }
1097
1098 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1099 {
1100 TCGTemp *ts = tcgv_i64_temp(arg);
1101
1102 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1103 if (ts->kind == TEMP_CONST) {
1104 tcg_gen_movi_i64(ret, ts->val);
1105 } else {
1106 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1107 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1108 }
1109 }
1110
1111 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1112 {
1113 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1114 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1115 }
1116
1117 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1118 {
1119 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1120 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1121 }
1122
1123 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1124 {
1125 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1126 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1127 }
1128
1129 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1130 {
1131 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1132 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1133 }
1134
1135 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1136 {
1137 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1138 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1139 }
1140
1141 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1142 {
1143 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1144 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1145 }
1146
1147 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1148 {
1149 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1150 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1151 }
1152
1153 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1154 {
1155 /* Since arg2 and ret have different types,
1156 they cannot be the same temporary */
1157 #if HOST_BIG_ENDIAN
1158 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1159 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1160 #else
1161 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1162 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1163 #endif
1164 }
1165
1166 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1167 {
1168 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset);
1169 }
1170
1171 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1172 {
1173 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset);
1174 }
1175
1176 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1177 {
1178 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1179 }
1180
1181 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1182 {
1183 #if HOST_BIG_ENDIAN
1184 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1185 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1186 #else
1187 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1188 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1189 #endif
1190 }
1191
1192 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1193 {
1194 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1),
1195 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2));
1196 }
1197
1198 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1199 {
1200 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1),
1201 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2));
1202 }
1203
1204 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1205 {
1206 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1207 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1208 }
1209
1210 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1211 {
1212 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1213 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1214 }
1215
1216 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1217 {
1218 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1219 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1220 }
1221
1222 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1223 {
1224 gen_helper_shl_i64(ret, arg1, arg2);
1225 }
1226
1227 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1228 {
1229 gen_helper_shr_i64(ret, arg1, arg2);
1230 }
1231
1232 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1233 {
1234 gen_helper_sar_i64(ret, arg1, arg2);
1235 }
1236
1237 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1238 {
1239 TCGv_i64 t0;
1240 TCGv_i32 t1;
1241
1242 t0 = tcg_temp_new_i64();
1243 t1 = tcg_temp_new_i32();
1244
1245 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1246 TCGV_LOW(arg1), TCGV_LOW(arg2));
1247
1248 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1249 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1250 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1251 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1252
1253 tcg_gen_mov_i64(ret, t0);
1254 tcg_temp_free_i64(t0);
1255 tcg_temp_free_i32(t1);
1256 }
1257
1258 #else
1259
1260 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1261 {
1262 tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1263 }
1264
1265 #endif /* TCG_TARGET_REG_SIZE == 32 */
1266
1267 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1268 {
1269 /* some cases can be optimized here */
1270 if (arg2 == 0) {
1271 tcg_gen_mov_i64(ret, arg1);
1272 } else if (TCG_TARGET_REG_BITS == 64) {
1273 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
1274 } else {
1275 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1276 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1277 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1278 }
1279 }
1280
1281 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1282 {
1283 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1284 /* Don't recurse with tcg_gen_neg_i64. */
1285 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1286 } else if (TCG_TARGET_REG_BITS == 64) {
1287 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
1288 } else {
1289 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1290 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
1291 TCGV_LOW(arg2), TCGV_HIGH(arg2));
1292 }
1293 }
1294
1295 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1296 {
1297 /* some cases can be optimized here */
1298 if (arg2 == 0) {
1299 tcg_gen_mov_i64(ret, arg1);
1300 } else if (TCG_TARGET_REG_BITS == 64) {
1301 tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2));
1302 } else {
1303 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1304 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1305 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1306 }
1307 }
1308
1309 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1310 {
1311 if (TCG_TARGET_REG_BITS == 32) {
1312 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1313 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1314 return;
1315 }
1316
1317 /* Some cases can be optimized here. */
1318 switch (arg2) {
1319 case 0:
1320 tcg_gen_movi_i64(ret, 0);
1321 return;
1322 case -1:
1323 tcg_gen_mov_i64(ret, arg1);
1324 return;
1325 case 0xff:
1326 /* Don't recurse with tcg_gen_ext8u_i64. */
1327 if (TCG_TARGET_HAS_ext8u_i64) {
1328 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1329 return;
1330 }
1331 break;
1332 case 0xffff:
1333 if (TCG_TARGET_HAS_ext16u_i64) {
1334 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1335 return;
1336 }
1337 break;
1338 case 0xffffffffu:
1339 if (TCG_TARGET_HAS_ext32u_i64) {
1340 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1341 return;
1342 }
1343 break;
1344 }
1345
1346 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
1347 }
1348
1349 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1350 {
1351 if (TCG_TARGET_REG_BITS == 32) {
1352 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1353 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1354 return;
1355 }
1356 /* Some cases can be optimized here. */
1357 if (arg2 == -1) {
1358 tcg_gen_movi_i64(ret, -1);
1359 } else if (arg2 == 0) {
1360 tcg_gen_mov_i64(ret, arg1);
1361 } else {
1362 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
1363 }
1364 }
1365
1366 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1367 {
1368 if (TCG_TARGET_REG_BITS == 32) {
1369 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1370 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1371 return;
1372 }
1373 /* Some cases can be optimized here. */
1374 if (arg2 == 0) {
1375 tcg_gen_mov_i64(ret, arg1);
1376 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1377 /* Don't recurse with tcg_gen_not_i64. */
1378 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1379 } else {
1380 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
1381 }
1382 }
1383
1384 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1385 unsigned c, bool right, bool arith)
1386 {
1387 tcg_debug_assert(c < 64);
1388 if (c == 0) {
1389 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1390 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1391 } else if (c >= 32) {
1392 c -= 32;
1393 if (right) {
1394 if (arith) {
1395 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1396 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1397 } else {
1398 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1399 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1400 }
1401 } else {
1402 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1403 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1404 }
1405 } else if (right) {
1406 if (TCG_TARGET_HAS_extract2_i32) {
1407 tcg_gen_extract2_i32(TCGV_LOW(ret),
1408 TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1409 } else {
1410 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1411 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1412 TCGV_HIGH(arg1), 32 - c, c);
1413 }
1414 if (arith) {
1415 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1416 } else {
1417 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1418 }
1419 } else {
1420 if (TCG_TARGET_HAS_extract2_i32) {
1421 tcg_gen_extract2_i32(TCGV_HIGH(ret),
1422 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
1423 } else {
1424 TCGv_i32 t0 = tcg_temp_new_i32();
1425 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1426 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1427 TCGV_HIGH(arg1), c, 32 - c);
1428 tcg_temp_free_i32(t0);
1429 }
1430 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1431 }
1432 }
1433
1434 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1435 {
1436 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1437 if (TCG_TARGET_REG_BITS == 32) {
1438 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1439 } else if (arg2 == 0) {
1440 tcg_gen_mov_i64(ret, arg1);
1441 } else {
1442 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
1443 }
1444 }
1445
1446 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1447 {
1448 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1449 if (TCG_TARGET_REG_BITS == 32) {
1450 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1451 } else if (arg2 == 0) {
1452 tcg_gen_mov_i64(ret, arg1);
1453 } else {
1454 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
1455 }
1456 }
1457
1458 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1459 {
1460 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1461 if (TCG_TARGET_REG_BITS == 32) {
1462 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1463 } else if (arg2 == 0) {
1464 tcg_gen_mov_i64(ret, arg1);
1465 } else {
1466 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
1467 }
1468 }
1469
1470 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1471 {
1472 if (cond == TCG_COND_ALWAYS) {
1473 tcg_gen_br(l);
1474 } else if (cond != TCG_COND_NEVER) {
1475 l->refs++;
1476 if (TCG_TARGET_REG_BITS == 32) {
1477 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1478 TCGV_HIGH(arg1), TCGV_LOW(arg2),
1479 TCGV_HIGH(arg2), cond, label_arg(l));
1480 } else {
1481 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1482 label_arg(l));
1483 }
1484 }
1485 }
1486
1487 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1488 {
1489 if (TCG_TARGET_REG_BITS == 64) {
1490 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1491 } else if (cond == TCG_COND_ALWAYS) {
1492 tcg_gen_br(l);
1493 } else if (cond != TCG_COND_NEVER) {
1494 l->refs++;
1495 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
1496 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1497 tcg_constant_i32(arg2),
1498 tcg_constant_i32(arg2 >> 32),
1499 cond, label_arg(l));
1500 }
1501 }
1502
1503 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1504 TCGv_i64 arg1, TCGv_i64 arg2)
1505 {
1506 if (cond == TCG_COND_ALWAYS) {
1507 tcg_gen_movi_i64(ret, 1);
1508 } else if (cond == TCG_COND_NEVER) {
1509 tcg_gen_movi_i64(ret, 0);
1510 } else {
1511 if (TCG_TARGET_REG_BITS == 32) {
1512 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1513 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1514 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1515 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1516 } else {
1517 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1518 }
1519 }
1520 }
1521
1522 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1523 TCGv_i64 arg1, int64_t arg2)
1524 {
1525 if (TCG_TARGET_REG_BITS == 64) {
1526 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1527 } else if (cond == TCG_COND_ALWAYS) {
1528 tcg_gen_movi_i64(ret, 1);
1529 } else if (cond == TCG_COND_NEVER) {
1530 tcg_gen_movi_i64(ret, 0);
1531 } else {
1532 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1533 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1534 tcg_constant_i32(arg2),
1535 tcg_constant_i32(arg2 >> 32), cond);
1536 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1537 }
1538 }
1539
1540 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1541 {
1542 if (arg2 == 0) {
1543 tcg_gen_movi_i64(ret, 0);
1544 } else if (is_power_of_2(arg2)) {
1545 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1546 } else {
1547 TCGv_i64 t0 = tcg_const_i64(arg2);
1548 tcg_gen_mul_i64(ret, arg1, t0);
1549 tcg_temp_free_i64(t0);
1550 }
1551 }
1552
1553 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1554 {
1555 if (TCG_TARGET_HAS_div_i64) {
1556 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1557 } else if (TCG_TARGET_HAS_div2_i64) {
1558 TCGv_i64 t0 = tcg_temp_new_i64();
1559 tcg_gen_sari_i64(t0, arg1, 63);
1560 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1561 tcg_temp_free_i64(t0);
1562 } else {
1563 gen_helper_div_i64(ret, arg1, arg2);
1564 }
1565 }
1566
1567 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1568 {
1569 if (TCG_TARGET_HAS_rem_i64) {
1570 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1571 } else if (TCG_TARGET_HAS_div_i64) {
1572 TCGv_i64 t0 = tcg_temp_new_i64();
1573 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1574 tcg_gen_mul_i64(t0, t0, arg2);
1575 tcg_gen_sub_i64(ret, arg1, t0);
1576 tcg_temp_free_i64(t0);
1577 } else if (TCG_TARGET_HAS_div2_i64) {
1578 TCGv_i64 t0 = tcg_temp_new_i64();
1579 tcg_gen_sari_i64(t0, arg1, 63);
1580 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1581 tcg_temp_free_i64(t0);
1582 } else {
1583 gen_helper_rem_i64(ret, arg1, arg2);
1584 }
1585 }
1586
1587 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1588 {
1589 if (TCG_TARGET_HAS_div_i64) {
1590 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1591 } else if (TCG_TARGET_HAS_div2_i64) {
1592 TCGv_i64 t0 = tcg_temp_new_i64();
1593 tcg_gen_movi_i64(t0, 0);
1594 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1595 tcg_temp_free_i64(t0);
1596 } else {
1597 gen_helper_divu_i64(ret, arg1, arg2);
1598 }
1599 }
1600
1601 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1602 {
1603 if (TCG_TARGET_HAS_rem_i64) {
1604 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1605 } else if (TCG_TARGET_HAS_div_i64) {
1606 TCGv_i64 t0 = tcg_temp_new_i64();
1607 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1608 tcg_gen_mul_i64(t0, t0, arg2);
1609 tcg_gen_sub_i64(ret, arg1, t0);
1610 tcg_temp_free_i64(t0);
1611 } else if (TCG_TARGET_HAS_div2_i64) {
1612 TCGv_i64 t0 = tcg_temp_new_i64();
1613 tcg_gen_movi_i64(t0, 0);
1614 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1615 tcg_temp_free_i64(t0);
1616 } else {
1617 gen_helper_remu_i64(ret, arg1, arg2);
1618 }
1619 }
1620
1621 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1622 {
1623 if (TCG_TARGET_REG_BITS == 32) {
1624 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1625 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1626 } else if (TCG_TARGET_HAS_ext8s_i64) {
1627 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1628 } else {
1629 tcg_gen_shli_i64(ret, arg, 56);
1630 tcg_gen_sari_i64(ret, ret, 56);
1631 }
1632 }
1633
1634 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1635 {
1636 if (TCG_TARGET_REG_BITS == 32) {
1637 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1638 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1639 } else if (TCG_TARGET_HAS_ext16s_i64) {
1640 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1641 } else {
1642 tcg_gen_shli_i64(ret, arg, 48);
1643 tcg_gen_sari_i64(ret, ret, 48);
1644 }
1645 }
1646
1647 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1648 {
1649 if (TCG_TARGET_REG_BITS == 32) {
1650 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1651 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1652 } else if (TCG_TARGET_HAS_ext32s_i64) {
1653 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1654 } else {
1655 tcg_gen_shli_i64(ret, arg, 32);
1656 tcg_gen_sari_i64(ret, ret, 32);
1657 }
1658 }
1659
1660 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1661 {
1662 if (TCG_TARGET_REG_BITS == 32) {
1663 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1664 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1665 } else if (TCG_TARGET_HAS_ext8u_i64) {
1666 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1667 } else {
1668 tcg_gen_andi_i64(ret, arg, 0xffu);
1669 }
1670 }
1671
1672 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1673 {
1674 if (TCG_TARGET_REG_BITS == 32) {
1675 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1676 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1677 } else if (TCG_TARGET_HAS_ext16u_i64) {
1678 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1679 } else {
1680 tcg_gen_andi_i64(ret, arg, 0xffffu);
1681 }
1682 }
1683
1684 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1685 {
1686 if (TCG_TARGET_REG_BITS == 32) {
1687 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1688 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1689 } else if (TCG_TARGET_HAS_ext32u_i64) {
1690 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1691 } else {
1692 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1693 }
1694 }
1695
1696 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1697 {
1698 /* Only one extension flag may be present. */
1699 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1700
1701 if (TCG_TARGET_REG_BITS == 32) {
1702 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
1703 if (flags & TCG_BSWAP_OS) {
1704 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1705 } else {
1706 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1707 }
1708 } else if (TCG_TARGET_HAS_bswap16_i64) {
1709 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
1710 } else {
1711 TCGv_i64 t0 = tcg_temp_new_i64();
1712 TCGv_i64 t1 = tcg_temp_new_i64();
1713
1714 tcg_gen_shri_i64(t0, arg, 8);
1715 if (!(flags & TCG_BSWAP_IZ)) {
1716 tcg_gen_ext8u_i64(t0, t0);
1717 }
1718
1719 if (flags & TCG_BSWAP_OS) {
1720 tcg_gen_shli_i64(t1, arg, 56);
1721 tcg_gen_sari_i64(t1, t1, 48);
1722 } else if (flags & TCG_BSWAP_OZ) {
1723 tcg_gen_ext8u_i64(t1, arg);
1724 tcg_gen_shli_i64(t1, t1, 8);
1725 } else {
1726 tcg_gen_shli_i64(t1, arg, 8);
1727 }
1728
1729 tcg_gen_or_i64(ret, t0, t1);
1730 tcg_temp_free_i64(t0);
1731 tcg_temp_free_i64(t1);
1732 }
1733 }
1734
1735 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1736 {
1737 /* Only one extension flag may be present. */
1738 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1739
1740 if (TCG_TARGET_REG_BITS == 32) {
1741 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1742 if (flags & TCG_BSWAP_OS) {
1743 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1744 } else {
1745 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1746 }
1747 } else if (TCG_TARGET_HAS_bswap32_i64) {
1748 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
1749 } else {
1750 TCGv_i64 t0 = tcg_temp_new_i64();
1751 TCGv_i64 t1 = tcg_temp_new_i64();
1752 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
1753
1754 /* arg = xxxxabcd */
1755 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */
1756 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */
1757 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */
1758 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */
1759 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */
1760
1761 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1762 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1763 if (flags & TCG_BSWAP_OS) {
1764 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */
1765 } else {
1766 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1767 }
1768 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba */
1769
1770 tcg_temp_free_i64(t0);
1771 tcg_temp_free_i64(t1);
1772 }
1773 }
1774
1775 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1776 {
1777 if (TCG_TARGET_REG_BITS == 32) {
1778 TCGv_i32 t0, t1;
1779 t0 = tcg_temp_new_i32();
1780 t1 = tcg_temp_new_i32();
1781
1782 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1783 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1784 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1785 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1786 tcg_temp_free_i32(t0);
1787 tcg_temp_free_i32(t1);
1788 } else if (TCG_TARGET_HAS_bswap64_i64) {
1789 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
1790 } else {
1791 TCGv_i64 t0 = tcg_temp_new_i64();
1792 TCGv_i64 t1 = tcg_temp_new_i64();
1793 TCGv_i64 t2 = tcg_temp_new_i64();
1794
1795 /* arg = abcdefgh */
1796 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1797 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */
1798 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */
1799 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */
1800 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */
1801 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */
1802
1803 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1804 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */
1805 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */
1806 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */
1807 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */
1808 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */
1809
1810 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */
1811 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */
1812 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */
1813
1814 tcg_temp_free_i64(t0);
1815 tcg_temp_free_i64(t1);
1816 tcg_temp_free_i64(t2);
1817 }
1818 }
1819
1820 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1821 {
1822 uint64_t m = 0x0000ffff0000ffffull;
1823 TCGv_i64 t0 = tcg_temp_new_i64();
1824 TCGv_i64 t1 = tcg_temp_new_i64();
1825
1826 /* See include/qemu/bitops.h, hswap64. */
1827 tcg_gen_rotli_i64(t1, arg, 32);
1828 tcg_gen_andi_i64(t0, t1, m);
1829 tcg_gen_shli_i64(t0, t0, 16);
1830 tcg_gen_shri_i64(t1, t1, 16);
1831 tcg_gen_andi_i64(t1, t1, m);
1832 tcg_gen_or_i64(ret, t0, t1);
1833
1834 tcg_temp_free_i64(t0);
1835 tcg_temp_free_i64(t1);
1836 }
1837
1838 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1839 {
1840 /* Swapping 2 32-bit elements is a rotate. */
1841 tcg_gen_rotli_i64(ret, arg, 32);
1842 }
1843
1844 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1845 {
1846 if (TCG_TARGET_REG_BITS == 32) {
1847 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1848 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1849 } else if (TCG_TARGET_HAS_not_i64) {
1850 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1851 } else {
1852 tcg_gen_xori_i64(ret, arg, -1);
1853 }
1854 }
1855
1856 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1857 {
1858 if (TCG_TARGET_REG_BITS == 32) {
1859 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1860 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1861 } else if (TCG_TARGET_HAS_andc_i64) {
1862 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1863 } else {
1864 TCGv_i64 t0 = tcg_temp_new_i64();
1865 tcg_gen_not_i64(t0, arg2);
1866 tcg_gen_and_i64(ret, arg1, t0);
1867 tcg_temp_free_i64(t0);
1868 }
1869 }
1870
1871 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1872 {
1873 if (TCG_TARGET_REG_BITS == 32) {
1874 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1875 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1876 } else if (TCG_TARGET_HAS_eqv_i64) {
1877 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1878 } else {
1879 tcg_gen_xor_i64(ret, arg1, arg2);
1880 tcg_gen_not_i64(ret, ret);
1881 }
1882 }
1883
1884 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1885 {
1886 if (TCG_TARGET_REG_BITS == 32) {
1887 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1888 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1889 } else if (TCG_TARGET_HAS_nand_i64) {
1890 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1891 } else {
1892 tcg_gen_and_i64(ret, arg1, arg2);
1893 tcg_gen_not_i64(ret, ret);
1894 }
1895 }
1896
1897 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1898 {
1899 if (TCG_TARGET_REG_BITS == 32) {
1900 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1901 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1902 } else if (TCG_TARGET_HAS_nor_i64) {
1903 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1904 } else {
1905 tcg_gen_or_i64(ret, arg1, arg2);
1906 tcg_gen_not_i64(ret, ret);
1907 }
1908 }
1909
1910 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1911 {
1912 if (TCG_TARGET_REG_BITS == 32) {
1913 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1914 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1915 } else if (TCG_TARGET_HAS_orc_i64) {
1916 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1917 } else {
1918 TCGv_i64 t0 = tcg_temp_new_i64();
1919 tcg_gen_not_i64(t0, arg2);
1920 tcg_gen_or_i64(ret, arg1, t0);
1921 tcg_temp_free_i64(t0);
1922 }
1923 }
1924
1925 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1926 {
1927 if (TCG_TARGET_HAS_clz_i64) {
1928 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1929 } else {
1930 gen_helper_clz_i64(ret, arg1, arg2);
1931 }
1932 }
1933
1934 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1935 {
1936 if (TCG_TARGET_REG_BITS == 32
1937 && TCG_TARGET_HAS_clz_i32
1938 && arg2 <= 0xffffffffu) {
1939 TCGv_i32 t = tcg_temp_new_i32();
1940 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
1941 tcg_gen_addi_i32(t, t, 32);
1942 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1943 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1944 tcg_temp_free_i32(t);
1945 } else {
1946 TCGv_i64 t0 = tcg_const_i64(arg2);
1947 tcg_gen_clz_i64(ret, arg1, t0);
1948 tcg_temp_free_i64(t0);
1949 }
1950 }
1951
1952 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1953 {
1954 if (TCG_TARGET_HAS_ctz_i64) {
1955 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1956 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1957 TCGv_i64 z, t = tcg_temp_new_i64();
1958
1959 if (TCG_TARGET_HAS_ctpop_i64) {
1960 tcg_gen_subi_i64(t, arg1, 1);
1961 tcg_gen_andc_i64(t, t, arg1);
1962 tcg_gen_ctpop_i64(t, t);
1963 } else {
1964 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1965 tcg_gen_neg_i64(t, arg1);
1966 tcg_gen_and_i64(t, t, arg1);
1967 tcg_gen_clzi_i64(t, t, 64);
1968 tcg_gen_xori_i64(t, t, 63);
1969 }
1970 z = tcg_constant_i64(0);
1971 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1972 tcg_temp_free_i64(t);
1973 tcg_temp_free_i64(z);
1974 } else {
1975 gen_helper_ctz_i64(ret, arg1, arg2);
1976 }
1977 }
1978
1979 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1980 {
1981 if (TCG_TARGET_REG_BITS == 32
1982 && TCG_TARGET_HAS_ctz_i32
1983 && arg2 <= 0xffffffffu) {
1984 TCGv_i32 t32 = tcg_temp_new_i32();
1985 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
1986 tcg_gen_addi_i32(t32, t32, 32);
1987 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1988 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1989 tcg_temp_free_i32(t32);
1990 } else if (!TCG_TARGET_HAS_ctz_i64
1991 && TCG_TARGET_HAS_ctpop_i64
1992 && arg2 == 64) {
1993 /* This equivalence has the advantage of not requiring a fixup. */
1994 TCGv_i64 t = tcg_temp_new_i64();
1995 tcg_gen_subi_i64(t, arg1, 1);
1996 tcg_gen_andc_i64(t, t, arg1);
1997 tcg_gen_ctpop_i64(ret, t);
1998 tcg_temp_free_i64(t);
1999 } else {
2000 TCGv_i64 t0 = tcg_const_i64(arg2);
2001 tcg_gen_ctz_i64(ret, arg1, t0);
2002 tcg_temp_free_i64(t0);
2003 }
2004 }
2005
2006 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
2007 {
2008 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
2009 TCGv_i64 t = tcg_temp_new_i64();
2010 tcg_gen_sari_i64(t, arg, 63);
2011 tcg_gen_xor_i64(t, t, arg);
2012 tcg_gen_clzi_i64(t, t, 64);
2013 tcg_gen_subi_i64(ret, t, 1);
2014 tcg_temp_free_i64(t);
2015 } else {
2016 gen_helper_clrsb_i64(ret, arg);
2017 }
2018 }
2019
2020 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
2021 {
2022 if (TCG_TARGET_HAS_ctpop_i64) {
2023 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
2024 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
2025 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2026 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2027 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
2028 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2029 } else {
2030 gen_helper_ctpop_i64(ret, arg1);
2031 }
2032 }
2033
2034 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2035 {
2036 if (TCG_TARGET_HAS_rot_i64) {
2037 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
2038 } else {
2039 TCGv_i64 t0, t1;
2040 t0 = tcg_temp_new_i64();
2041 t1 = tcg_temp_new_i64();
2042 tcg_gen_shl_i64(t0, arg1, arg2);
2043 tcg_gen_subfi_i64(t1, 64, arg2);
2044 tcg_gen_shr_i64(t1, arg1, t1);
2045 tcg_gen_or_i64(ret, t0, t1);
2046 tcg_temp_free_i64(t0);
2047 tcg_temp_free_i64(t1);
2048 }
2049 }
2050
2051 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2052 {
2053 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2054 /* some cases can be optimized here */
2055 if (arg2 == 0) {
2056 tcg_gen_mov_i64(ret, arg1);
2057 } else if (TCG_TARGET_HAS_rot_i64) {
2058 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
2059 } else {
2060 TCGv_i64 t0, t1;
2061 t0 = tcg_temp_new_i64();
2062 t1 = tcg_temp_new_i64();
2063 tcg_gen_shli_i64(t0, arg1, arg2);
2064 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
2065 tcg_gen_or_i64(ret, t0, t1);
2066 tcg_temp_free_i64(t0);
2067 tcg_temp_free_i64(t1);
2068 }
2069 }
2070
2071 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2072 {
2073 if (TCG_TARGET_HAS_rot_i64) {
2074 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
2075 } else {
2076 TCGv_i64 t0, t1;
2077 t0 = tcg_temp_new_i64();
2078 t1 = tcg_temp_new_i64();
2079 tcg_gen_shr_i64(t0, arg1, arg2);
2080 tcg_gen_subfi_i64(t1, 64, arg2);
2081 tcg_gen_shl_i64(t1, arg1, t1);
2082 tcg_gen_or_i64(ret, t0, t1);
2083 tcg_temp_free_i64(t0);
2084 tcg_temp_free_i64(t1);
2085 }
2086 }
2087
2088 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2089 {
2090 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2091 /* some cases can be optimized here */
2092 if (arg2 == 0) {
2093 tcg_gen_mov_i64(ret, arg1);
2094 } else {
2095 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2096 }
2097 }
2098
2099 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2100 unsigned int ofs, unsigned int len)
2101 {
2102 uint64_t mask;
2103 TCGv_i64 t1;
2104
2105 tcg_debug_assert(ofs < 64);
2106 tcg_debug_assert(len > 0);
2107 tcg_debug_assert(len <= 64);
2108 tcg_debug_assert(ofs + len <= 64);
2109
2110 if (len == 64) {
2111 tcg_gen_mov_i64(ret, arg2);
2112 return;
2113 }
2114 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2115 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2116 return;
2117 }
2118
2119 if (TCG_TARGET_REG_BITS == 32) {
2120 if (ofs >= 32) {
2121 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2122 TCGV_LOW(arg2), ofs - 32, len);
2123 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2124 return;
2125 }
2126 if (ofs + len <= 32) {
2127 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2128 TCGV_LOW(arg2), ofs, len);
2129 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2130 return;
2131 }
2132 }
2133
2134 t1 = tcg_temp_new_i64();
2135
2136 if (TCG_TARGET_HAS_extract2_i64) {
2137 if (ofs + len == 64) {
2138 tcg_gen_shli_i64(t1, arg1, len);
2139 tcg_gen_extract2_i64(ret, t1, arg2, len);
2140 goto done;
2141 }
2142 if (ofs == 0) {
2143 tcg_gen_extract2_i64(ret, arg1, arg2, len);
2144 tcg_gen_rotli_i64(ret, ret, len);
2145 goto done;
2146 }
2147 }
2148
2149 mask = (1ull << len) - 1;
2150 if (ofs + len < 64) {
2151 tcg_gen_andi_i64(t1, arg2, mask);
2152 tcg_gen_shli_i64(t1, t1, ofs);
2153 } else {
2154 tcg_gen_shli_i64(t1, arg2, ofs);
2155 }
2156 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2157 tcg_gen_or_i64(ret, ret, t1);
2158 done:
2159 tcg_temp_free_i64(t1);
2160 }
2161
2162 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2163 unsigned int ofs, unsigned int len)
2164 {
2165 tcg_debug_assert(ofs < 64);
2166 tcg_debug_assert(len > 0);
2167 tcg_debug_assert(len <= 64);
2168 tcg_debug_assert(ofs + len <= 64);
2169
2170 if (ofs + len == 64) {
2171 tcg_gen_shli_i64(ret, arg, ofs);
2172 } else if (ofs == 0) {
2173 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2174 } else if (TCG_TARGET_HAS_deposit_i64
2175 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2176 TCGv_i64 zero = tcg_constant_i64(0);
2177 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2178 } else {
2179 if (TCG_TARGET_REG_BITS == 32) {
2180 if (ofs >= 32) {
2181 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2182 ofs - 32, len);
2183 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2184 return;
2185 }
2186 if (ofs + len <= 32) {
2187 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2188 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2189 return;
2190 }
2191 }
2192 /* To help two-operand hosts we prefer to zero-extend first,
2193 which allows ARG to stay live. */
2194 switch (len) {
2195 case 32:
2196 if (TCG_TARGET_HAS_ext32u_i64) {
2197 tcg_gen_ext32u_i64(ret, arg);
2198 tcg_gen_shli_i64(ret, ret, ofs);
2199 return;
2200 }
2201 break;
2202 case 16:
2203 if (TCG_TARGET_HAS_ext16u_i64) {
2204 tcg_gen_ext16u_i64(ret, arg);
2205 tcg_gen_shli_i64(ret, ret, ofs);
2206 return;
2207 }
2208 break;
2209 case 8:
2210 if (TCG_TARGET_HAS_ext8u_i64) {
2211 tcg_gen_ext8u_i64(ret, arg);
2212 tcg_gen_shli_i64(ret, ret, ofs);
2213 return;
2214 }
2215 break;
2216 }
2217 /* Otherwise prefer zero-extension over AND for code size. */
2218 switch (ofs + len) {
2219 case 32:
2220 if (TCG_TARGET_HAS_ext32u_i64) {
2221 tcg_gen_shli_i64(ret, arg, ofs);
2222 tcg_gen_ext32u_i64(ret, ret);
2223 return;
2224 }
2225 break;
2226 case 16:
2227 if (TCG_TARGET_HAS_ext16u_i64) {
2228 tcg_gen_shli_i64(ret, arg, ofs);
2229 tcg_gen_ext16u_i64(ret, ret);
2230 return;
2231 }
2232 break;
2233 case 8:
2234 if (TCG_TARGET_HAS_ext8u_i64) {
2235 tcg_gen_shli_i64(ret, arg, ofs);
2236 tcg_gen_ext8u_i64(ret, ret);
2237 return;
2238 }
2239 break;
2240 }
2241 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2242 tcg_gen_shli_i64(ret, ret, ofs);
2243 }
2244 }
2245
2246 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2247 unsigned int ofs, unsigned int len)
2248 {
2249 tcg_debug_assert(ofs < 64);
2250 tcg_debug_assert(len > 0);
2251 tcg_debug_assert(len <= 64);
2252 tcg_debug_assert(ofs + len <= 64);
2253
2254 /* Canonicalize certain special cases, even if extract is supported. */
2255 if (ofs + len == 64) {
2256 tcg_gen_shri_i64(ret, arg, 64 - len);
2257 return;
2258 }
2259 if (ofs == 0) {
2260 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2261 return;
2262 }
2263
2264 if (TCG_TARGET_REG_BITS == 32) {
2265 /* Look for a 32-bit extract within one of the two words. */
2266 if (ofs >= 32) {
2267 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2268 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2269 return;
2270 }
2271 if (ofs + len <= 32) {
2272 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2273 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2274 return;
2275 }
2276 /* The field is split across two words. One double-word
2277 shift is better than two double-word shifts. */
2278 goto do_shift_and;
2279 }
2280
2281 if (TCG_TARGET_HAS_extract_i64
2282 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2283 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2284 return;
2285 }
2286
2287 /* Assume that zero-extension, if available, is cheaper than a shift. */
2288 switch (ofs + len) {
2289 case 32:
2290 if (TCG_TARGET_HAS_ext32u_i64) {
2291 tcg_gen_ext32u_i64(ret, arg);
2292 tcg_gen_shri_i64(ret, ret, ofs);
2293 return;
2294 }
2295 break;
2296 case 16:
2297 if (TCG_TARGET_HAS_ext16u_i64) {
2298 tcg_gen_ext16u_i64(ret, arg);
2299 tcg_gen_shri_i64(ret, ret, ofs);
2300 return;
2301 }
2302 break;
2303 case 8:
2304 if (TCG_TARGET_HAS_ext8u_i64) {
2305 tcg_gen_ext8u_i64(ret, arg);
2306 tcg_gen_shri_i64(ret, ret, ofs);
2307 return;
2308 }
2309 break;
2310 }
2311
2312 /* ??? Ideally we'd know what values are available for immediate AND.
2313 Assume that 8 bits are available, plus the special cases of 16 and 32,
2314 so that we get ext8u, ext16u, and ext32u. */
2315 switch (len) {
2316 case 1 ... 8: case 16: case 32:
2317 do_shift_and:
2318 tcg_gen_shri_i64(ret, arg, ofs);
2319 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2320 break;
2321 default:
2322 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2323 tcg_gen_shri_i64(ret, ret, 64 - len);
2324 break;
2325 }
2326 }
2327
2328 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2329 unsigned int ofs, unsigned int len)
2330 {
2331 tcg_debug_assert(ofs < 64);
2332 tcg_debug_assert(len > 0);
2333 tcg_debug_assert(len <= 64);
2334 tcg_debug_assert(ofs + len <= 64);
2335
2336 /* Canonicalize certain special cases, even if sextract is supported. */
2337 if (ofs + len == 64) {
2338 tcg_gen_sari_i64(ret, arg, 64 - len);
2339 return;
2340 }
2341 if (ofs == 0) {
2342 switch (len) {
2343 case 32:
2344 tcg_gen_ext32s_i64(ret, arg);
2345 return;
2346 case 16:
2347 tcg_gen_ext16s_i64(ret, arg);
2348 return;
2349 case 8:
2350 tcg_gen_ext8s_i64(ret, arg);
2351 return;
2352 }
2353 }
2354
2355 if (TCG_TARGET_REG_BITS == 32) {
2356 /* Look for a 32-bit extract within one of the two words. */
2357 if (ofs >= 32) {
2358 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2359 } else if (ofs + len <= 32) {
2360 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2361 } else if (ofs == 0) {
2362 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2363 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2364 return;
2365 } else if (len > 32) {
2366 TCGv_i32 t = tcg_temp_new_i32();
2367 /* Extract the bits for the high word normally. */
2368 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2369 /* Shift the field down for the low part. */
2370 tcg_gen_shri_i64(ret, arg, ofs);
2371 /* Overwrite the shift into the high part. */
2372 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2373 tcg_temp_free_i32(t);
2374 return;
2375 } else {
2376 /* Shift the field down for the low part, such that the
2377 field sits at the MSB. */
2378 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2379 /* Shift the field down from the MSB, sign extending. */
2380 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2381 }
2382 /* Sign-extend the field from 32 bits. */
2383 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2384 return;
2385 }
2386
2387 if (TCG_TARGET_HAS_sextract_i64
2388 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2389 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2390 return;
2391 }
2392
2393 /* Assume that sign-extension, if available, is cheaper than a shift. */
2394 switch (ofs + len) {
2395 case 32:
2396 if (TCG_TARGET_HAS_ext32s_i64) {
2397 tcg_gen_ext32s_i64(ret, arg);
2398 tcg_gen_sari_i64(ret, ret, ofs);
2399 return;
2400 }
2401 break;
2402 case 16:
2403 if (TCG_TARGET_HAS_ext16s_i64) {
2404 tcg_gen_ext16s_i64(ret, arg);
2405 tcg_gen_sari_i64(ret, ret, ofs);
2406 return;
2407 }
2408 break;
2409 case 8:
2410 if (TCG_TARGET_HAS_ext8s_i64) {
2411 tcg_gen_ext8s_i64(ret, arg);
2412 tcg_gen_sari_i64(ret, ret, ofs);
2413 return;
2414 }
2415 break;
2416 }
2417 switch (len) {
2418 case 32:
2419 if (TCG_TARGET_HAS_ext32s_i64) {
2420 tcg_gen_shri_i64(ret, arg, ofs);
2421 tcg_gen_ext32s_i64(ret, ret);
2422 return;
2423 }
2424 break;
2425 case 16:
2426 if (TCG_TARGET_HAS_ext16s_i64) {
2427 tcg_gen_shri_i64(ret, arg, ofs);
2428 tcg_gen_ext16s_i64(ret, ret);
2429 return;
2430 }
2431 break;
2432 case 8:
2433 if (TCG_TARGET_HAS_ext8s_i64) {
2434 tcg_gen_shri_i64(ret, arg, ofs);
2435 tcg_gen_ext8s_i64(ret, ret);
2436 return;
2437 }
2438 break;
2439 }
2440 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2441 tcg_gen_sari_i64(ret, ret, 64 - len);
2442 }
2443
2444 /*
2445 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2446 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2447 */
2448 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2449 unsigned int ofs)
2450 {
2451 tcg_debug_assert(ofs <= 64);
2452 if (ofs == 0) {
2453 tcg_gen_mov_i64(ret, al);
2454 } else if (ofs == 64) {
2455 tcg_gen_mov_i64(ret, ah);
2456 } else if (al == ah) {
2457 tcg_gen_rotri_i64(ret, al, ofs);
2458 } else if (TCG_TARGET_HAS_extract2_i64) {
2459 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2460 } else {
2461 TCGv_i64 t0 = tcg_temp_new_i64();
2462 tcg_gen_shri_i64(t0, al, ofs);
2463 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2464 tcg_temp_free_i64(t0);
2465 }
2466 }
2467
2468 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2469 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2470 {
2471 if (cond == TCG_COND_ALWAYS) {
2472 tcg_gen_mov_i64(ret, v1);
2473 } else if (cond == TCG_COND_NEVER) {
2474 tcg_gen_mov_i64(ret, v2);
2475 } else if (TCG_TARGET_REG_BITS == 32) {
2476 TCGv_i32 t0 = tcg_temp_new_i32();
2477 TCGv_i32 t1 = tcg_temp_new_i32();
2478 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2479 TCGV_LOW(c1), TCGV_HIGH(c1),
2480 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2481
2482 if (TCG_TARGET_HAS_movcond_i32) {
2483 tcg_gen_movi_i32(t1, 0);
2484 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2485 TCGV_LOW(v1), TCGV_LOW(v2));
2486 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2487 TCGV_HIGH(v1), TCGV_HIGH(v2));
2488 } else {
2489 tcg_gen_neg_i32(t0, t0);
2490
2491 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2492 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2493 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2494
2495 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2496 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2497 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2498 }
2499 tcg_temp_free_i32(t0);
2500 tcg_temp_free_i32(t1);
2501 } else if (TCG_TARGET_HAS_movcond_i64) {
2502 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2503 } else {
2504 TCGv_i64 t0 = tcg_temp_new_i64();
2505 TCGv_i64 t1 = tcg_temp_new_i64();
2506 tcg_gen_setcond_i64(cond, t0, c1, c2);
2507 tcg_gen_neg_i64(t0, t0);
2508 tcg_gen_and_i64(t1, v1, t0);
2509 tcg_gen_andc_i64(ret, v2, t0);
2510 tcg_gen_or_i64(ret, ret, t1);
2511 tcg_temp_free_i64(t0);
2512 tcg_temp_free_i64(t1);
2513 }
2514 }
2515
2516 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2517 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2518 {
2519 if (TCG_TARGET_HAS_add2_i64) {
2520 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2521 } else {
2522 TCGv_i64 t0 = tcg_temp_new_i64();
2523 TCGv_i64 t1 = tcg_temp_new_i64();
2524 tcg_gen_add_i64(t0, al, bl);
2525 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2526 tcg_gen_add_i64(rh, ah, bh);
2527 tcg_gen_add_i64(rh, rh, t1);
2528 tcg_gen_mov_i64(rl, t0);
2529 tcg_temp_free_i64(t0);
2530 tcg_temp_free_i64(t1);
2531 }
2532 }
2533
2534 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2535 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2536 {
2537 if (TCG_TARGET_HAS_sub2_i64) {
2538 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2539 } else {
2540 TCGv_i64 t0 = tcg_temp_new_i64();
2541 TCGv_i64 t1 = tcg_temp_new_i64();
2542 tcg_gen_sub_i64(t0, al, bl);
2543 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2544 tcg_gen_sub_i64(rh, ah, bh);
2545 tcg_gen_sub_i64(rh, rh, t1);
2546 tcg_gen_mov_i64(rl, t0);
2547 tcg_temp_free_i64(t0);
2548 tcg_temp_free_i64(t1);
2549 }
2550 }
2551
2552 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2553 {
2554 if (TCG_TARGET_HAS_mulu2_i64) {
2555 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2556 } else if (TCG_TARGET_HAS_muluh_i64) {
2557 TCGv_i64 t = tcg_temp_new_i64();
2558 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2559 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2560 tcg_gen_mov_i64(rl, t);
2561 tcg_temp_free_i64(t);
2562 } else {
2563 TCGv_i64 t0 = tcg_temp_new_i64();
2564 tcg_gen_mul_i64(t0, arg1, arg2);
2565 gen_helper_muluh_i64(rh, arg1, arg2);
2566 tcg_gen_mov_i64(rl, t0);
2567 tcg_temp_free_i64(t0);
2568 }
2569 }
2570
2571 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2572 {
2573 if (TCG_TARGET_HAS_muls2_i64) {
2574 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2575 } else if (TCG_TARGET_HAS_mulsh_i64) {
2576 TCGv_i64 t = tcg_temp_new_i64();
2577 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2578 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2579 tcg_gen_mov_i64(rl, t);
2580 tcg_temp_free_i64(t);
2581 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2582 TCGv_i64 t0 = tcg_temp_new_i64();
2583 TCGv_i64 t1 = tcg_temp_new_i64();
2584 TCGv_i64 t2 = tcg_temp_new_i64();
2585 TCGv_i64 t3 = tcg_temp_new_i64();
2586 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2587 /* Adjust for negative inputs. */
2588 tcg_gen_sari_i64(t2, arg1, 63);
2589 tcg_gen_sari_i64(t3, arg2, 63);
2590 tcg_gen_and_i64(t2, t2, arg2);
2591 tcg_gen_and_i64(t3, t3, arg1);
2592 tcg_gen_sub_i64(rh, t1, t2);
2593 tcg_gen_sub_i64(rh, rh, t3);
2594 tcg_gen_mov_i64(rl, t0);
2595 tcg_temp_free_i64(t0);
2596 tcg_temp_free_i64(t1);
2597 tcg_temp_free_i64(t2);
2598 tcg_temp_free_i64(t3);
2599 } else {
2600 TCGv_i64 t0 = tcg_temp_new_i64();
2601 tcg_gen_mul_i64(t0, arg1, arg2);
2602 gen_helper_mulsh_i64(rh, arg1, arg2);
2603 tcg_gen_mov_i64(rl, t0);
2604 tcg_temp_free_i64(t0);
2605 }
2606 }
2607
2608 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2609 {
2610 TCGv_i64 t0 = tcg_temp_new_i64();
2611 TCGv_i64 t1 = tcg_temp_new_i64();
2612 TCGv_i64 t2 = tcg_temp_new_i64();
2613 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2614 /* Adjust for negative input for the signed arg1. */
2615 tcg_gen_sari_i64(t2, arg1, 63);
2616 tcg_gen_and_i64(t2, t2, arg2);
2617 tcg_gen_sub_i64(rh, t1, t2);
2618 tcg_gen_mov_i64(rl, t0);
2619 tcg_temp_free_i64(t0);
2620 tcg_temp_free_i64(t1);
2621 tcg_temp_free_i64(t2);
2622 }
2623
2624 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2625 {
2626 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2627 }
2628
2629 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2630 {
2631 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2632 }
2633
2634 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2635 {
2636 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2637 }
2638
2639 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2640 {
2641 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2642 }
2643
2644 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2645 {
2646 TCGv_i64 t = tcg_temp_new_i64();
2647
2648 tcg_gen_sari_i64(t, a, 63);
2649 tcg_gen_xor_i64(ret, a, t);
2650 tcg_gen_sub_i64(ret, ret, t);
2651 tcg_temp_free_i64(t);
2652 }
2653
2654 /* Size changing operations. */
2655
2656 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2657 {
2658 if (TCG_TARGET_REG_BITS == 32) {
2659 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2660 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2661 tcg_gen_op2(INDEX_op_extrl_i64_i32,
2662 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2663 } else {
2664 tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2665 }
2666 }
2667
2668 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2669 {
2670 if (TCG_TARGET_REG_BITS == 32) {
2671 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2672 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2673 tcg_gen_op2(INDEX_op_extrh_i64_i32,
2674 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2675 } else {
2676 TCGv_i64 t = tcg_temp_new_i64();
2677 tcg_gen_shri_i64(t, arg, 32);
2678 tcg_gen_mov_i32(ret, (TCGv_i32)t);
2679 tcg_temp_free_i64(t);
2680 }
2681 }
2682
2683 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2684 {
2685 if (TCG_TARGET_REG_BITS == 32) {
2686 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2687 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2688 } else {
2689 tcg_gen_op2(INDEX_op_extu_i32_i64,
2690 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2691 }
2692 }
2693
2694 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2695 {
2696 if (TCG_TARGET_REG_BITS == 32) {
2697 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2698 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2699 } else {
2700 tcg_gen_op2(INDEX_op_ext_i32_i64,
2701 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2702 }
2703 }
2704
2705 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2706 {
2707 TCGv_i64 tmp;
2708
2709 if (TCG_TARGET_REG_BITS == 32) {
2710 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2711 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2712 return;
2713 }
2714
2715 tmp = tcg_temp_new_i64();
2716 /* These extensions are only needed for type correctness.
2717 We may be able to do better given target specific information. */
2718 tcg_gen_extu_i32_i64(tmp, high);
2719 tcg_gen_extu_i32_i64(dest, low);
2720 /* If deposit is available, use it. Otherwise use the extra
2721 knowledge that we have of the zero-extensions above. */
2722 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2723 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2724 } else {
2725 tcg_gen_shli_i64(tmp, tmp, 32);
2726 tcg_gen_or_i64(dest, dest, tmp);
2727 }
2728 tcg_temp_free_i64(tmp);
2729 }
2730
2731 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2732 {
2733 if (TCG_TARGET_REG_BITS == 32) {
2734 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2735 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2736 } else {
2737 tcg_gen_extrl_i64_i32(lo, arg);
2738 tcg_gen_extrh_i64_i32(hi, arg);
2739 }
2740 }
2741
2742 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2743 {
2744 tcg_gen_ext32u_i64(lo, arg);
2745 tcg_gen_shri_i64(hi, arg, 32);
2746 }
2747
2748 /* QEMU specific operations. */
2749
2750 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
2751 {
2752 /*
2753 * Let the jit code return the read-only version of the
2754 * TranslationBlock, so that we minimize the pc-relative
2755 * distance of the address of the exit_tb code to TB.
2756 * This will improve utilization of pc-relative address loads.
2757 *
2758 * TODO: Move this to translator_loop, so that all const
2759 * TranslationBlock pointers refer to read-only memory.
2760 * This requires coordination with targets that do not use
2761 * the translator_loop.
2762 */
2763 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
2764
2765 if (tb == NULL) {
2766 tcg_debug_assert(idx == 0);
2767 } else if (idx <= TB_EXIT_IDXMAX) {
2768 #ifdef CONFIG_DEBUG_TCG
2769 /* This is an exit following a goto_tb. Verify that we have
2770 seen this numbered exit before, via tcg_gen_goto_tb. */
2771 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2772 #endif
2773 } else {
2774 /* This is an exit via the exitreq label. */
2775 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2776 }
2777
2778 plugin_gen_disable_mem_helpers();
2779 tcg_gen_op1i(INDEX_op_exit_tb, val);
2780 }
2781
2782 void tcg_gen_goto_tb(unsigned idx)
2783 {
2784 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
2785 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB));
2786 /* We only support two chained exits. */
2787 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2788 #ifdef CONFIG_DEBUG_TCG
2789 /* Verify that we haven't seen this numbered exit before. */
2790 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2791 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2792 #endif
2793 plugin_gen_disable_mem_helpers();
2794 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2795 }
2796
2797 void tcg_gen_lookup_and_goto_ptr(void)
2798 {
2799 TCGv_ptr ptr;
2800
2801 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) {
2802 tcg_gen_exit_tb(NULL, 0);
2803 return;
2804 }
2805
2806 plugin_gen_disable_mem_helpers();
2807 ptr = tcg_temp_new_ptr();
2808 gen_helper_lookup_tb_ptr(ptr, cpu_env);
2809 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2810 tcg_temp_free_ptr(ptr);
2811 }
2812
2813 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
2814 {
2815 /* Trigger the asserts within as early as possible. */
2816 unsigned a_bits = get_alignment_bits(op);
2817
2818 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
2819 if (a_bits == (op & MO_SIZE)) {
2820 op = (op & ~MO_AMASK) | MO_ALIGN;
2821 }
2822
2823 switch (op & MO_SIZE) {
2824 case MO_8:
2825 op &= ~MO_BSWAP;
2826 break;
2827 case MO_16:
2828 break;
2829 case MO_32:
2830 if (!is64) {
2831 op &= ~MO_SIGN;
2832 }
2833 break;
2834 case MO_64:
2835 if (is64) {
2836 op &= ~MO_SIGN;
2837 break;
2838 }
2839 /* fall through */
2840 default:
2841 g_assert_not_reached();
2842 }
2843 if (st) {
2844 op &= ~MO_SIGN;
2845 }
2846 return op;
2847 }
2848
2849 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2850 MemOp memop, TCGArg idx)
2851 {
2852 MemOpIdx oi = make_memop_idx(memop, idx);
2853 #if TARGET_LONG_BITS == 32
2854 tcg_gen_op3i_i32(opc, val, addr, oi);
2855 #else
2856 if (TCG_TARGET_REG_BITS == 32) {
2857 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2858 } else {
2859 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2860 }
2861 #endif
2862 }
2863
2864 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2865 MemOp memop, TCGArg idx)
2866 {
2867 MemOpIdx oi = make_memop_idx(memop, idx);
2868 #if TARGET_LONG_BITS == 32
2869 if (TCG_TARGET_REG_BITS == 32) {
2870 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2871 } else {
2872 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2873 }
2874 #else
2875 if (TCG_TARGET_REG_BITS == 32) {
2876 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2877 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2878 } else {
2879 tcg_gen_op3i_i64(opc, val, addr, oi);
2880 }
2881 #endif
2882 }
2883
2884 static void tcg_gen_req_mo(TCGBar type)
2885 {
2886 #ifdef TCG_GUEST_DEFAULT_MO
2887 type &= TCG_GUEST_DEFAULT_MO;
2888 #endif
2889 type &= ~TCG_TARGET_DEFAULT_MO;
2890 if (type) {
2891 tcg_gen_mb(type | TCG_BAR_SC);
2892 }
2893 }
2894
2895 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr)
2896 {
2897 #ifdef CONFIG_PLUGIN
2898 if (tcg_ctx->plugin_insn != NULL) {
2899 /* Save a copy of the vaddr for use after a load. */
2900 TCGv temp = tcg_temp_new();
2901 tcg_gen_mov_tl(temp, vaddr);
2902 return temp;
2903 }
2904 #endif
2905 return vaddr;
2906 }
2907
2908 static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi,
2909 enum qemu_plugin_mem_rw rw)
2910 {
2911 #ifdef CONFIG_PLUGIN
2912 if (tcg_ctx->plugin_insn != NULL) {
2913 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
2914 plugin_gen_empty_mem_callback(vaddr, info);
2915 tcg_temp_free(vaddr);
2916 }
2917 #endif
2918 }
2919
2920 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2921 {
2922 MemOp orig_memop;
2923 MemOpIdx oi;
2924
2925 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2926 memop = tcg_canonicalize_memop(memop, 0, 0);
2927 oi = make_memop_idx(memop, idx);
2928
2929 orig_memop = memop;
2930 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2931 memop &= ~MO_BSWAP;
2932 /* The bswap primitive benefits from zero-extended input. */
2933 if ((memop & MO_SSIZE) == MO_SW) {
2934 memop &= ~MO_SIGN;
2935 }
2936 }
2937
2938 addr = plugin_prep_mem_callbacks(addr);
2939 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2940 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
2941
2942 if ((orig_memop ^ memop) & MO_BSWAP) {
2943 switch (orig_memop & MO_SIZE) {
2944 case MO_16:
2945 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
2946 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2947 : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
2948 break;
2949 case MO_32:
2950 tcg_gen_bswap32_i32(val, val);
2951 break;
2952 default:
2953 g_assert_not_reached();
2954 }
2955 }
2956 }
2957
2958 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2959 {
2960 TCGv_i32 swap = NULL;
2961 MemOpIdx oi;
2962
2963 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2964 memop = tcg_canonicalize_memop(memop, 0, 1);
2965 oi = make_memop_idx(memop, idx);
2966
2967 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2968 swap = tcg_temp_new_i32();
2969 switch (memop & MO_SIZE) {
2970 case MO_16:
2971 tcg_gen_bswap16_i32(swap, val, 0);
2972 break;
2973 case MO_32:
2974 tcg_gen_bswap32_i32(swap, val);
2975 break;
2976 default:
2977 g_assert_not_reached();
2978 }
2979 val = swap;
2980 memop &= ~MO_BSWAP;
2981 }
2982
2983 addr = plugin_prep_mem_callbacks(addr);
2984 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
2985 gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
2986 } else {
2987 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2988 }
2989 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
2990
2991 if (swap) {
2992 tcg_temp_free_i32(swap);
2993 }
2994 }
2995
2996 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2997 {
2998 MemOp orig_memop;
2999 MemOpIdx oi;
3000
3001 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
3002 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
3003 if (memop & MO_SIGN) {
3004 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
3005 } else {
3006 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
3007 }
3008 return;
3009 }
3010
3011 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
3012 memop = tcg_canonicalize_memop(memop, 1, 0);
3013 oi = make_memop_idx(memop, idx);
3014
3015 orig_memop = memop;
3016 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3017 memop &= ~MO_BSWAP;
3018 /* The bswap primitive benefits from zero-extended input. */
3019 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
3020 memop &= ~MO_SIGN;
3021 }
3022 }
3023
3024 addr = plugin_prep_mem_callbacks(addr);
3025 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
3026 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
3027
3028 if ((orig_memop ^ memop) & MO_BSWAP) {
3029 int flags = (orig_memop & MO_SIGN
3030 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
3031 : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
3032 switch (orig_memop & MO_SIZE) {
3033 case MO_16:
3034 tcg_gen_bswap16_i64(val, val, flags);
3035 break;
3036 case MO_32:
3037 tcg_gen_bswap32_i64(val, val, flags);
3038 break;
3039 case MO_64:
3040 tcg_gen_bswap64_i64(val, val);
3041 break;
3042 default:
3043 g_assert_not_reached();
3044 }
3045 }
3046 }
3047
3048 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
3049 {
3050 TCGv_i64 swap = NULL;
3051 MemOpIdx oi;
3052
3053 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
3054 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
3055 return;
3056 }
3057
3058 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
3059 memop = tcg_canonicalize_memop(memop, 1, 1);
3060 oi = make_memop_idx(memop, idx);
3061
3062 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3063 swap = tcg_temp_new_i64();
3064 switch (memop & MO_SIZE) {
3065 case MO_16:
3066 tcg_gen_bswap16_i64(swap, val, 0);
3067 break;
3068 case MO_32:
3069 tcg_gen_bswap32_i64(swap, val, 0);
3070 break;
3071 case MO_64:
3072 tcg_gen_bswap64_i64(swap, val);
3073 break;
3074 default:
3075 g_assert_not_reached();
3076 }
3077 val = swap;
3078 memop &= ~MO_BSWAP;
3079 }
3080
3081 addr = plugin_prep_mem_callbacks(addr);
3082 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
3083 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
3084
3085 if (swap) {
3086 tcg_temp_free_i64(swap);
3087 }
3088 }
3089
3090 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
3091 {
3092 switch (opc & MO_SSIZE) {
3093 case MO_SB:
3094 tcg_gen_ext8s_i32(ret, val);
3095 break;
3096 case MO_UB:
3097 tcg_gen_ext8u_i32(ret, val);
3098 break;
3099 case MO_SW:
3100 tcg_gen_ext16s_i32(ret, val);
3101 break;
3102 case MO_UW:
3103 tcg_gen_ext16u_i32(ret, val);
3104 break;
3105 default:
3106 tcg_gen_mov_i32(ret, val);
3107 break;
3108 }
3109 }
3110
3111 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
3112 {
3113 switch (opc & MO_SSIZE) {
3114 case MO_SB:
3115 tcg_gen_ext8s_i64(ret, val);
3116 break;
3117 case MO_UB:
3118 tcg_gen_ext8u_i64(ret, val);
3119 break;
3120 case MO_SW:
3121 tcg_gen_ext16s_i64(ret, val);
3122 break;
3123 case MO_UW:
3124 tcg_gen_ext16u_i64(ret, val);
3125 break;
3126 case MO_SL:
3127 tcg_gen_ext32s_i64(ret, val);
3128 break;
3129 case MO_UL:
3130 tcg_gen_ext32u_i64(ret, val);
3131 break;
3132 default:
3133 tcg_gen_mov_i64(ret, val);
3134 break;
3135 }
3136 }
3137
3138 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3139 TCGv_i32, TCGv_i32, TCGv_i32);
3140 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3141 TCGv_i64, TCGv_i64, TCGv_i32);
3142 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3143 TCGv_i32, TCGv_i32);
3144 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3145 TCGv_i64, TCGv_i32);
3146
3147 #ifdef CONFIG_ATOMIC64
3148 # define WITH_ATOMIC64(X) X,
3149 #else
3150 # define WITH_ATOMIC64(X)
3151 #endif
3152
3153 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
3154 [MO_8] = gen_helper_atomic_cmpxchgb,
3155 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3156 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3157 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3158 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
3159 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3160 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
3161 };
3162
3163 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3164 TCGv_i32 newv, TCGArg idx, MemOp memop)
3165 {
3166 memop = tcg_canonicalize_memop(memop, 0, 0);
3167
3168 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
3169 TCGv_i32 t1 = tcg_temp_new_i32();
3170 TCGv_i32 t2 = tcg_temp_new_i32();
3171
3172 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3173
3174 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3175 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3176 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3177 tcg_temp_free_i32(t2);
3178
3179 if (memop & MO_SIGN) {
3180 tcg_gen_ext_i32(retv, t1, memop);
3181 } else {
3182 tcg_gen_mov_i32(retv, t1);
3183 }
3184 tcg_temp_free_i32(t1);
3185 } else {
3186 gen_atomic_cx_i32 gen;
3187 MemOpIdx oi;
3188
3189 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3190 tcg_debug_assert(gen != NULL);
3191
3192 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3193 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3194
3195 if (memop & MO_SIGN) {
3196 tcg_gen_ext_i32(retv, retv, memop);
3197 }
3198 }
3199 }
3200
3201 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3202 TCGv_i64 newv, TCGArg idx, MemOp memop)
3203 {
3204 memop = tcg_canonicalize_memop(memop, 1, 0);
3205
3206 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
3207 TCGv_i64 t1 = tcg_temp_new_i64();
3208 TCGv_i64 t2 = tcg_temp_new_i64();
3209
3210 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
3211
3212 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3213 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3214 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3215 tcg_temp_free_i64(t2);
3216
3217 if (memop & MO_SIGN) {
3218 tcg_gen_ext_i64(retv, t1, memop);
3219 } else {
3220 tcg_gen_mov_i64(retv, t1);
3221 }
3222 tcg_temp_free_i64(t1);
3223 } else if ((memop & MO_SIZE) == MO_64) {
3224 #ifdef CONFIG_ATOMIC64
3225 gen_atomic_cx_i64 gen;
3226 MemOpIdx oi;
3227
3228 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3229 tcg_debug_assert(gen != NULL);
3230
3231 oi = make_memop_idx(memop, idx);
3232 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3233 #else
3234 gen_helper_exit_atomic(cpu_env);
3235 /* Produce a result, so that we have a well-formed opcode stream
3236 with respect to uses of the result in the (dead) code following. */
3237 tcg_gen_movi_i64(retv, 0);
3238 #endif /* CONFIG_ATOMIC64 */
3239 } else {
3240 TCGv_i32 c32 = tcg_temp_new_i32();
3241 TCGv_i32 n32 = tcg_temp_new_i32();
3242 TCGv_i32 r32 = tcg_temp_new_i32();
3243
3244 tcg_gen_extrl_i64_i32(c32, cmpv);
3245 tcg_gen_extrl_i64_i32(n32, newv);
3246 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3247 tcg_temp_free_i32(c32);
3248 tcg_temp_free_i32(n32);
3249
3250 tcg_gen_extu_i32_i64(retv, r32);
3251 tcg_temp_free_i32(r32);
3252
3253 if (memop & MO_SIGN) {
3254 tcg_gen_ext_i64(retv, retv, memop);
3255 }
3256 }
3257 }
3258
3259 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3260 TCGArg idx, MemOp memop, bool new_val,
3261 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3262 {
3263 TCGv_i32 t1 = tcg_temp_new_i32();
3264 TCGv_i32 t2 = tcg_temp_new_i32();
3265
3266 memop = tcg_canonicalize_memop(memop, 0, 0);
3267
3268 tcg_gen_qemu_ld_i32(t1, addr, idx, memop);
3269 tcg_gen_ext_i32(t2, val, memop);
3270 gen(t2, t1, t2);
3271 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3272
3273 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3274 tcg_temp_free_i32(t1);
3275 tcg_temp_free_i32(t2);
3276 }
3277
3278 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3279 TCGArg idx, MemOp memop, void * const table[])
3280 {
3281 gen_atomic_op_i32 gen;
3282 MemOpIdx oi;
3283
3284 memop = tcg_canonicalize_memop(memop, 0, 0);
3285
3286 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3287 tcg_debug_assert(gen != NULL);
3288
3289 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3290 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3291
3292 if (memop & MO_SIGN) {
3293 tcg_gen_ext_i32(ret, ret, memop);
3294 }
3295 }
3296
3297 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3298 TCGArg idx, MemOp memop, bool new_val,
3299 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3300 {
3301 TCGv_i64 t1 = tcg_temp_new_i64();
3302 TCGv_i64 t2 = tcg_temp_new_i64();
3303
3304 memop = tcg_canonicalize_memop(memop, 1, 0);
3305
3306 tcg_gen_qemu_ld_i64(t1, addr, idx, memop);
3307 tcg_gen_ext_i64(t2, val, memop);
3308 gen(t2, t1, t2);
3309 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3310
3311 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3312 tcg_temp_free_i64(t1);
3313 tcg_temp_free_i64(t2);
3314 }
3315
3316 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3317 TCGArg idx, MemOp memop, void * const table[])
3318 {
3319 memop = tcg_canonicalize_memop(memop, 1, 0);
3320
3321 if ((memop & MO_SIZE) == MO_64) {
3322 #ifdef CONFIG_ATOMIC64
3323 gen_atomic_op_i64 gen;
3324 MemOpIdx oi;
3325
3326 gen = table[memop & (MO_SIZE | MO_BSWAP)];
3327 tcg_debug_assert(gen != NULL);
3328
3329 oi = make_memop_idx(memop & ~MO_SIGN, idx);
3330 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3331 #else
3332 gen_helper_exit_atomic(cpu_env);
3333 /* Produce a result, so that we have a well-formed opcode stream
3334 with respect to uses of the result in the (dead) code following. */
3335 tcg_gen_movi_i64(ret, 0);
3336 #endif /* CONFIG_ATOMIC64 */
3337 } else {
3338 TCGv_i32 v32 = tcg_temp_new_i32();
3339 TCGv_i32 r32 = tcg_temp_new_i32();
3340
3341 tcg_gen_extrl_i64_i32(v32, val);
3342 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3343 tcg_temp_free_i32(v32);
3344
3345 tcg_gen_extu_i32_i64(ret, r32);
3346 tcg_temp_free_i32(r32);
3347
3348 if (memop & MO_SIGN) {
3349 tcg_gen_ext_i64(ret, ret, memop);
3350 }
3351 }
3352 }
3353
3354 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3355 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
3356 [MO_8] = gen_helper_atomic_##NAME##b, \
3357 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3358 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3359 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3360 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
3361 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3362 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
3363 }; \
3364 void tcg_gen_atomic_##NAME##_i32 \
3365 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \
3366 { \
3367 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
3368 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3369 } else { \
3370 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3371 tcg_gen_##OP##_i32); \
3372 } \
3373 } \
3374 void tcg_gen_atomic_##NAME##_i64 \
3375 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \
3376 { \
3377 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
3378 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3379 } else { \
3380 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3381 tcg_gen_##OP##_i64); \
3382 } \
3383 }
3384
3385 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3386 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3387 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3388 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3389 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3390 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3391 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3392 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3393
3394 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3395 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3396 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3397 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3398 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3399 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3400 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3401 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3402
3403 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3404 {
3405 tcg_gen_mov_i32(r, b);
3406 }
3407
3408 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3409 {
3410 tcg_gen_mov_i64(r, b);
3411 }
3412
3413 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3414
3415 #undef GEN_ATOMIC_HELPER
QEmu