tcg/tcg-op.c

   1 /*
   2  * Tiny Code Generator for QEMU
   3  *
   4  * Copyright (c) 2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "qemu-common.h"
  27 #include "cpu.h"
  28 #include "exec/exec-all.h"
  29 #include "tcg.h"
  30 #include "tcg-op.h"
  31 #include "trace-tcg.h"
  32 #include "trace/mem.h"
  33
  34 /* Reduce the number of ifdefs below.  This assumes that all uses of
  35    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
  36    the compiler can eliminate.  */
  37 #if TCG_TARGET_REG_BITS == 64
  38 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
  39 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
  40 #define TCGV_LOW  TCGV_LOW_link_error
  41 #define TCGV_HIGH TCGV_HIGH_link_error
  42 #endif
  43
  44 /* Note that this is optimized for sequential allocation during translate.
  45    Up to and including filling in the forward link immediately.  We'll do
  46    proper termination of the end of the list after we finish translation.  */
  47
  48 static void tcg_emit_op(TCGContext *ctx, TCGOpcode opc, int args)
  49 {
  50     int oi = ctx->gen_next_op_idx;
  51     int ni = oi + 1;
  52     int pi = oi - 1;
  53
  54     tcg_debug_assert(oi < OPC_BUF_SIZE);
  55     ctx->gen_op_buf[0].prev = oi;
  56     ctx->gen_next_op_idx = ni;
  57
  58     ctx->gen_op_buf[oi] = (TCGOp){
  59         .opc = opc,
  60         .args = args,
  61         .prev = pi,
  62         .next = ni
  63     };
  64 }
  65
  66 void tcg_gen_op1(TCGContext *ctx, TCGOpcode opc, TCGArg a1)
  67 {
  68     int pi = ctx->gen_next_parm_idx;
  69
  70     tcg_debug_assert(pi + 1 <= OPPARAM_BUF_SIZE);
  71     ctx->gen_next_parm_idx = pi + 1;
  72     ctx->gen_opparam_buf[pi] = a1;
  73
  74     tcg_emit_op(ctx, opc, pi);
  75 }
  76
  77 void tcg_gen_op2(TCGContext *ctx, TCGOpcode opc, TCGArg a1, TCGArg a2)
  78 {
  79     int pi = ctx->gen_next_parm_idx;
  80
  81     tcg_debug_assert(pi + 2 <= OPPARAM_BUF_SIZE);
  82     ctx->gen_next_parm_idx = pi + 2;
  83     ctx->gen_opparam_buf[pi + 0] = a1;
  84     ctx->gen_opparam_buf[pi + 1] = a2;
  85
  86     tcg_emit_op(ctx, opc, pi);
  87 }
  88
  89 void tcg_gen_op3(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
  90                  TCGArg a2, TCGArg a3)
  91 {
  92     int pi = ctx->gen_next_parm_idx;
  93
  94     tcg_debug_assert(pi + 3 <= OPPARAM_BUF_SIZE);
  95     ctx->gen_next_parm_idx = pi + 3;
  96     ctx->gen_opparam_buf[pi + 0] = a1;
  97     ctx->gen_opparam_buf[pi + 1] = a2;
  98     ctx->gen_opparam_buf[pi + 2] = a3;
  99
 100     tcg_emit_op(ctx, opc, pi);
 101 }
 102
 103 void tcg_gen_op4(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
 104                  TCGArg a2, TCGArg a3, TCGArg a4)
 105 {
 106     int pi = ctx->gen_next_parm_idx;
 107
 108     tcg_debug_assert(pi + 4 <= OPPARAM_BUF_SIZE);
 109     ctx->gen_next_parm_idx = pi + 4;
 110     ctx->gen_opparam_buf[pi + 0] = a1;
 111     ctx->gen_opparam_buf[pi + 1] = a2;
 112     ctx->gen_opparam_buf[pi + 2] = a3;
 113     ctx->gen_opparam_buf[pi + 3] = a4;
 114
 115     tcg_emit_op(ctx, opc, pi);
 116 }
 117
 118 void tcg_gen_op5(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
 119                  TCGArg a2, TCGArg a3, TCGArg a4, TCGArg a5)
 120 {
 121     int pi = ctx->gen_next_parm_idx;
 122
 123     tcg_debug_assert(pi + 5 <= OPPARAM_BUF_SIZE);
 124     ctx->gen_next_parm_idx = pi + 5;
 125     ctx->gen_opparam_buf[pi + 0] = a1;
 126     ctx->gen_opparam_buf[pi + 1] = a2;
 127     ctx->gen_opparam_buf[pi + 2] = a3;
 128     ctx->gen_opparam_buf[pi + 3] = a4;
 129     ctx->gen_opparam_buf[pi + 4] = a5;
 130
 131     tcg_emit_op(ctx, opc, pi);
 132 }
 133
 134 void tcg_gen_op6(TCGContext *ctx, TCGOpcode opc, TCGArg a1, TCGArg a2,
 135                  TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6)
 136 {
 137     int pi = ctx->gen_next_parm_idx;
 138
 139     tcg_debug_assert(pi + 6 <= OPPARAM_BUF_SIZE);
 140     ctx->gen_next_parm_idx = pi + 6;
 141     ctx->gen_opparam_buf[pi + 0] = a1;
 142     ctx->gen_opparam_buf[pi + 1] = a2;
 143     ctx->gen_opparam_buf[pi + 2] = a3;
 144     ctx->gen_opparam_buf[pi + 3] = a4;
 145     ctx->gen_opparam_buf[pi + 4] = a5;
 146     ctx->gen_opparam_buf[pi + 5] = a6;
 147
 148     tcg_emit_op(ctx, opc, pi);
 149 }
 150
 151 void tcg_gen_mb(TCGBar mb_type)
 152 {
 153     if (parallel_cpus) {
 154         tcg_gen_op1(&tcg_ctx, INDEX_op_mb, mb_type);
 155     }
 156 }
 157
 158 /* 32 bit ops */
 159
 160 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 161 {
 162     /* some cases can be optimized here */
 163     if (arg2 == 0) {
 164         tcg_gen_mov_i32(ret, arg1);
 165     } else {
 166         TCGv_i32 t0 = tcg_const_i32(arg2);
 167         tcg_gen_add_i32(ret, arg1, t0);
 168         tcg_temp_free_i32(t0);
 169     }
 170 }
 171
 172 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
 173 {
 174     if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
 175         /* Don't recurse with tcg_gen_neg_i32.  */
 176         tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
 177     } else {
 178         TCGv_i32 t0 = tcg_const_i32(arg1);
 179         tcg_gen_sub_i32(ret, t0, arg2);
 180         tcg_temp_free_i32(t0);
 181     }
 182 }
 183
 184 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 185 {
 186     /* some cases can be optimized here */
 187     if (arg2 == 0) {
 188         tcg_gen_mov_i32(ret, arg1);
 189     } else {
 190         TCGv_i32 t0 = tcg_const_i32(arg2);
 191         tcg_gen_sub_i32(ret, arg1, t0);
 192         tcg_temp_free_i32(t0);
 193     }
 194 }
 195
 196 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 197 {
 198     TCGv_i32 t0;
 199     /* Some cases can be optimized here.  */
 200     switch (arg2) {
 201     case 0:
 202         tcg_gen_movi_i32(ret, 0);
 203         return;
 204     case 0xffffffffu:
 205         tcg_gen_mov_i32(ret, arg1);
 206         return;
 207     case 0xffu:
 208         /* Don't recurse with tcg_gen_ext8u_i32.  */
 209         if (TCG_TARGET_HAS_ext8u_i32) {
 210             tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
 211             return;
 212         }
 213         break;
 214     case 0xffffu:
 215         if (TCG_TARGET_HAS_ext16u_i32) {
 216             tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
 217             return;
 218         }
 219         break;
 220     }
 221     t0 = tcg_const_i32(arg2);
 222     tcg_gen_and_i32(ret, arg1, t0);
 223     tcg_temp_free_i32(t0);
 224 }
 225
 226 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 227 {
 228     /* Some cases can be optimized here.  */
 229     if (arg2 == -1) {
 230         tcg_gen_movi_i32(ret, -1);
 231     } else if (arg2 == 0) {
 232         tcg_gen_mov_i32(ret, arg1);
 233     } else {
 234         TCGv_i32 t0 = tcg_const_i32(arg2);
 235         tcg_gen_or_i32(ret, arg1, t0);
 236         tcg_temp_free_i32(t0);
 237     }
 238 }
 239
 240 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 241 {
 242     /* Some cases can be optimized here.  */
 243     if (arg2 == 0) {
 244         tcg_gen_mov_i32(ret, arg1);
 245     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
 246         /* Don't recurse with tcg_gen_not_i32.  */
 247         tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
 248     } else {
 249         TCGv_i32 t0 = tcg_const_i32(arg2);
 250         tcg_gen_xor_i32(ret, arg1, t0);
 251         tcg_temp_free_i32(t0);
 252     }
 253 }
 254
 255 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 256 {
 257     tcg_debug_assert(arg2 < 32);
 258     if (arg2 == 0) {
 259         tcg_gen_mov_i32(ret, arg1);
 260     } else {
 261         TCGv_i32 t0 = tcg_const_i32(arg2);
 262         tcg_gen_shl_i32(ret, arg1, t0);
 263         tcg_temp_free_i32(t0);
 264     }
 265 }
 266
 267 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 268 {
 269     tcg_debug_assert(arg2 < 32);
 270     if (arg2 == 0) {
 271         tcg_gen_mov_i32(ret, arg1);
 272     } else {
 273         TCGv_i32 t0 = tcg_const_i32(arg2);
 274         tcg_gen_shr_i32(ret, arg1, t0);
 275         tcg_temp_free_i32(t0);
 276     }
 277 }
 278
 279 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 280 {
 281     tcg_debug_assert(arg2 < 32);
 282     if (arg2 == 0) {
 283         tcg_gen_mov_i32(ret, arg1);
 284     } else {
 285         TCGv_i32 t0 = tcg_const_i32(arg2);
 286         tcg_gen_sar_i32(ret, arg1, t0);
 287         tcg_temp_free_i32(t0);
 288     }
 289 }
 290
 291 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
 292 {
 293     if (cond == TCG_COND_ALWAYS) {
 294         tcg_gen_br(l);
 295     } else if (cond != TCG_COND_NEVER) {
 296         tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
 297     }
 298 }
 299
 300 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
 301 {
 302     if (cond == TCG_COND_ALWAYS) {
 303         tcg_gen_br(l);
 304     } else if (cond != TCG_COND_NEVER) {
 305         TCGv_i32 t0 = tcg_const_i32(arg2);
 306         tcg_gen_brcond_i32(cond, arg1, t0, l);
 307         tcg_temp_free_i32(t0);
 308     }
 309 }
 310
 311 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
 312                          TCGv_i32 arg1, TCGv_i32 arg2)
 313 {
 314     if (cond == TCG_COND_ALWAYS) {
 315         tcg_gen_movi_i32(ret, 1);
 316     } else if (cond == TCG_COND_NEVER) {
 317         tcg_gen_movi_i32(ret, 0);
 318     } else {
 319         tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
 320     }
 321 }
 322
 323 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
 324                           TCGv_i32 arg1, int32_t arg2)
 325 {
 326     TCGv_i32 t0 = tcg_const_i32(arg2);
 327     tcg_gen_setcond_i32(cond, ret, arg1, t0);
 328     tcg_temp_free_i32(t0);
 329 }
 330
 331 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 332 {
 333     TCGv_i32 t0 = tcg_const_i32(arg2);
 334     tcg_gen_mul_i32(ret, arg1, t0);
 335     tcg_temp_free_i32(t0);
 336 }
 337
 338 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 339 {
 340     if (TCG_TARGET_HAS_div_i32) {
 341         tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
 342     } else if (TCG_TARGET_HAS_div2_i32) {
 343         TCGv_i32 t0 = tcg_temp_new_i32();
 344         tcg_gen_sari_i32(t0, arg1, 31);
 345         tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
 346         tcg_temp_free_i32(t0);
 347     } else {
 348         gen_helper_div_i32(ret, arg1, arg2);
 349     }
 350 }
 351
 352 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 353 {
 354     if (TCG_TARGET_HAS_rem_i32) {
 355         tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
 356     } else if (TCG_TARGET_HAS_div_i32) {
 357         TCGv_i32 t0 = tcg_temp_new_i32();
 358         tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
 359         tcg_gen_mul_i32(t0, t0, arg2);
 360         tcg_gen_sub_i32(ret, arg1, t0);
 361         tcg_temp_free_i32(t0);
 362     } else if (TCG_TARGET_HAS_div2_i32) {
 363         TCGv_i32 t0 = tcg_temp_new_i32();
 364         tcg_gen_sari_i32(t0, arg1, 31);
 365         tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
 366         tcg_temp_free_i32(t0);
 367     } else {
 368         gen_helper_rem_i32(ret, arg1, arg2);
 369     }
 370 }
 371
 372 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 373 {
 374     if (TCG_TARGET_HAS_div_i32) {
 375         tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
 376     } else if (TCG_TARGET_HAS_div2_i32) {
 377         TCGv_i32 t0 = tcg_temp_new_i32();
 378         tcg_gen_movi_i32(t0, 0);
 379         tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
 380         tcg_temp_free_i32(t0);
 381     } else {
 382         gen_helper_divu_i32(ret, arg1, arg2);
 383     }
 384 }
 385
 386 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 387 {
 388     if (TCG_TARGET_HAS_rem_i32) {
 389         tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
 390     } else if (TCG_TARGET_HAS_div_i32) {
 391         TCGv_i32 t0 = tcg_temp_new_i32();
 392         tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
 393         tcg_gen_mul_i32(t0, t0, arg2);
 394         tcg_gen_sub_i32(ret, arg1, t0);
 395         tcg_temp_free_i32(t0);
 396     } else if (TCG_TARGET_HAS_div2_i32) {
 397         TCGv_i32 t0 = tcg_temp_new_i32();
 398         tcg_gen_movi_i32(t0, 0);
 399         tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
 400         tcg_temp_free_i32(t0);
 401     } else {
 402         gen_helper_remu_i32(ret, arg1, arg2);
 403     }
 404 }
 405
 406 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 407 {
 408     if (TCG_TARGET_HAS_andc_i32) {
 409         tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
 410     } else {
 411         TCGv_i32 t0 = tcg_temp_new_i32();
 412         tcg_gen_not_i32(t0, arg2);
 413         tcg_gen_and_i32(ret, arg1, t0);
 414         tcg_temp_free_i32(t0);
 415     }
 416 }
 417
 418 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 419 {
 420     if (TCG_TARGET_HAS_eqv_i32) {
 421         tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
 422     } else {
 423         tcg_gen_xor_i32(ret, arg1, arg2);
 424         tcg_gen_not_i32(ret, ret);
 425     }
 426 }
 427
 428 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 429 {
 430     if (TCG_TARGET_HAS_nand_i32) {
 431         tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
 432     } else {
 433         tcg_gen_and_i32(ret, arg1, arg2);
 434         tcg_gen_not_i32(ret, ret);
 435     }
 436 }
 437
 438 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 439 {
 440     if (TCG_TARGET_HAS_nor_i32) {
 441         tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
 442     } else {
 443         tcg_gen_or_i32(ret, arg1, arg2);
 444         tcg_gen_not_i32(ret, ret);
 445     }
 446 }
 447
 448 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 449 {
 450     if (TCG_TARGET_HAS_orc_i32) {
 451         tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
 452     } else {
 453         TCGv_i32 t0 = tcg_temp_new_i32();
 454         tcg_gen_not_i32(t0, arg2);
 455         tcg_gen_or_i32(ret, arg1, t0);
 456         tcg_temp_free_i32(t0);
 457     }
 458 }
 459
 460 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 461 {
 462     if (TCG_TARGET_HAS_rot_i32) {
 463         tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
 464     } else {
 465         TCGv_i32 t0, t1;
 466
 467         t0 = tcg_temp_new_i32();
 468         t1 = tcg_temp_new_i32();
 469         tcg_gen_shl_i32(t0, arg1, arg2);
 470         tcg_gen_subfi_i32(t1, 32, arg2);
 471         tcg_gen_shr_i32(t1, arg1, t1);
 472         tcg_gen_or_i32(ret, t0, t1);
 473         tcg_temp_free_i32(t0);
 474         tcg_temp_free_i32(t1);
 475     }
 476 }
 477
 478 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 479 {
 480     tcg_debug_assert(arg2 < 32);
 481     /* some cases can be optimized here */
 482     if (arg2 == 0) {
 483         tcg_gen_mov_i32(ret, arg1);
 484     } else if (TCG_TARGET_HAS_rot_i32) {
 485         TCGv_i32 t0 = tcg_const_i32(arg2);
 486         tcg_gen_rotl_i32(ret, arg1, t0);
 487         tcg_temp_free_i32(t0);
 488     } else {
 489         TCGv_i32 t0, t1;
 490         t0 = tcg_temp_new_i32();
 491         t1 = tcg_temp_new_i32();
 492         tcg_gen_shli_i32(t0, arg1, arg2);
 493         tcg_gen_shri_i32(t1, arg1, 32 - arg2);
 494         tcg_gen_or_i32(ret, t0, t1);
 495         tcg_temp_free_i32(t0);
 496         tcg_temp_free_i32(t1);
 497     }
 498 }
 499
 500 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 501 {
 502     if (TCG_TARGET_HAS_rot_i32) {
 503         tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
 504     } else {
 505         TCGv_i32 t0, t1;
 506
 507         t0 = tcg_temp_new_i32();
 508         t1 = tcg_temp_new_i32();
 509         tcg_gen_shr_i32(t0, arg1, arg2);
 510         tcg_gen_subfi_i32(t1, 32, arg2);
 511         tcg_gen_shl_i32(t1, arg1, t1);
 512         tcg_gen_or_i32(ret, t0, t1);
 513         tcg_temp_free_i32(t0);
 514         tcg_temp_free_i32(t1);
 515     }
 516 }
 517
 518 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 519 {
 520     tcg_debug_assert(arg2 < 32);
 521     /* some cases can be optimized here */
 522     if (arg2 == 0) {
 523         tcg_gen_mov_i32(ret, arg1);
 524     } else {
 525         tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
 526     }
 527 }
 528
 529 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
 530                          unsigned int ofs, unsigned int len)
 531 {
 532     uint32_t mask;
 533     TCGv_i32 t1;
 534
 535     tcg_debug_assert(ofs < 32);
 536     tcg_debug_assert(len <= 32);
 537     tcg_debug_assert(ofs + len <= 32);
 538
 539     if (ofs == 0 && len == 32) {
 540         tcg_gen_mov_i32(ret, arg2);
 541         return;
 542     }
 543     if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
 544         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
 545         return;
 546     }
 547
 548     mask = (1u << len) - 1;
 549     t1 = tcg_temp_new_i32();
 550
 551     if (ofs + len < 32) {
 552         tcg_gen_andi_i32(t1, arg2, mask);
 553         tcg_gen_shli_i32(t1, t1, ofs);
 554     } else {
 555         tcg_gen_shli_i32(t1, arg2, ofs);
 556     }
 557     tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
 558     tcg_gen_or_i32(ret, ret, t1);
 559
 560     tcg_temp_free_i32(t1);
 561 }
 562
 563 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
 564                          TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
 565 {
 566     if (cond == TCG_COND_ALWAYS) {
 567         tcg_gen_mov_i32(ret, v1);
 568     } else if (cond == TCG_COND_NEVER) {
 569         tcg_gen_mov_i32(ret, v2);
 570     } else if (TCG_TARGET_HAS_movcond_i32) {
 571         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
 572     } else {
 573         TCGv_i32 t0 = tcg_temp_new_i32();
 574         TCGv_i32 t1 = tcg_temp_new_i32();
 575         tcg_gen_setcond_i32(cond, t0, c1, c2);
 576         tcg_gen_neg_i32(t0, t0);
 577         tcg_gen_and_i32(t1, v1, t0);
 578         tcg_gen_andc_i32(ret, v2, t0);
 579         tcg_gen_or_i32(ret, ret, t1);
 580         tcg_temp_free_i32(t0);
 581         tcg_temp_free_i32(t1);
 582     }
 583 }
 584
 585 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
 586                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
 587 {
 588     if (TCG_TARGET_HAS_add2_i32) {
 589         tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
 590     } else {
 591         TCGv_i64 t0 = tcg_temp_new_i64();
 592         TCGv_i64 t1 = tcg_temp_new_i64();
 593         tcg_gen_concat_i32_i64(t0, al, ah);
 594         tcg_gen_concat_i32_i64(t1, bl, bh);
 595         tcg_gen_add_i64(t0, t0, t1);
 596         tcg_gen_extr_i64_i32(rl, rh, t0);
 597         tcg_temp_free_i64(t0);
 598         tcg_temp_free_i64(t1);
 599     }
 600 }
 601
 602 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
 603                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
 604 {
 605     if (TCG_TARGET_HAS_sub2_i32) {
 606         tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
 607     } else {
 608         TCGv_i64 t0 = tcg_temp_new_i64();
 609         TCGv_i64 t1 = tcg_temp_new_i64();
 610         tcg_gen_concat_i32_i64(t0, al, ah);
 611         tcg_gen_concat_i32_i64(t1, bl, bh);
 612         tcg_gen_sub_i64(t0, t0, t1);
 613         tcg_gen_extr_i64_i32(rl, rh, t0);
 614         tcg_temp_free_i64(t0);
 615         tcg_temp_free_i64(t1);
 616     }
 617 }
 618
 619 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 620 {
 621     if (TCG_TARGET_HAS_mulu2_i32) {
 622         tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
 623     } else if (TCG_TARGET_HAS_muluh_i32) {
 624         TCGv_i32 t = tcg_temp_new_i32();
 625         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
 626         tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
 627         tcg_gen_mov_i32(rl, t);
 628         tcg_temp_free_i32(t);
 629     } else {
 630         TCGv_i64 t0 = tcg_temp_new_i64();
 631         TCGv_i64 t1 = tcg_temp_new_i64();
 632         tcg_gen_extu_i32_i64(t0, arg1);
 633         tcg_gen_extu_i32_i64(t1, arg2);
 634         tcg_gen_mul_i64(t0, t0, t1);
 635         tcg_gen_extr_i64_i32(rl, rh, t0);
 636         tcg_temp_free_i64(t0);
 637         tcg_temp_free_i64(t1);
 638     }
 639 }
 640
 641 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 642 {
 643     if (TCG_TARGET_HAS_muls2_i32) {
 644         tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
 645     } else if (TCG_TARGET_HAS_mulsh_i32) {
 646         TCGv_i32 t = tcg_temp_new_i32();
 647         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
 648         tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
 649         tcg_gen_mov_i32(rl, t);
 650         tcg_temp_free_i32(t);
 651     } else if (TCG_TARGET_REG_BITS == 32) {
 652         TCGv_i32 t0 = tcg_temp_new_i32();
 653         TCGv_i32 t1 = tcg_temp_new_i32();
 654         TCGv_i32 t2 = tcg_temp_new_i32();
 655         TCGv_i32 t3 = tcg_temp_new_i32();
 656         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
 657         /* Adjust for negative inputs.  */
 658         tcg_gen_sari_i32(t2, arg1, 31);
 659         tcg_gen_sari_i32(t3, arg2, 31);
 660         tcg_gen_and_i32(t2, t2, arg2);
 661         tcg_gen_and_i32(t3, t3, arg1);
 662         tcg_gen_sub_i32(rh, t1, t2);
 663         tcg_gen_sub_i32(rh, rh, t3);
 664         tcg_gen_mov_i32(rl, t0);
 665         tcg_temp_free_i32(t0);
 666         tcg_temp_free_i32(t1);
 667         tcg_temp_free_i32(t2);
 668         tcg_temp_free_i32(t3);
 669     } else {
 670         TCGv_i64 t0 = tcg_temp_new_i64();
 671         TCGv_i64 t1 = tcg_temp_new_i64();
 672         tcg_gen_ext_i32_i64(t0, arg1);
 673         tcg_gen_ext_i32_i64(t1, arg2);
 674         tcg_gen_mul_i64(t0, t0, t1);
 675         tcg_gen_extr_i64_i32(rl, rh, t0);
 676         tcg_temp_free_i64(t0);
 677         tcg_temp_free_i64(t1);
 678     }
 679 }
 680
 681 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
 682 {
 683     if (TCG_TARGET_HAS_ext8s_i32) {
 684         tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
 685     } else {
 686         tcg_gen_shli_i32(ret, arg, 24);
 687         tcg_gen_sari_i32(ret, ret, 24);
 688     }
 689 }
 690
 691 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
 692 {
 693     if (TCG_TARGET_HAS_ext16s_i32) {
 694         tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
 695     } else {
 696         tcg_gen_shli_i32(ret, arg, 16);
 697         tcg_gen_sari_i32(ret, ret, 16);
 698     }
 699 }
 700
 701 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
 702 {
 703     if (TCG_TARGET_HAS_ext8u_i32) {
 704         tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
 705     } else {
 706         tcg_gen_andi_i32(ret, arg, 0xffu);
 707     }
 708 }
 709
 710 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
 711 {
 712     if (TCG_TARGET_HAS_ext16u_i32) {
 713         tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
 714     } else {
 715         tcg_gen_andi_i32(ret, arg, 0xffffu);
 716     }
 717 }
 718
 719 /* Note: we assume the two high bytes are set to zero */
 720 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
 721 {
 722     if (TCG_TARGET_HAS_bswap16_i32) {
 723         tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
 724     } else {
 725         TCGv_i32 t0 = tcg_temp_new_i32();
 726
 727         tcg_gen_ext8u_i32(t0, arg);
 728         tcg_gen_shli_i32(t0, t0, 8);
 729         tcg_gen_shri_i32(ret, arg, 8);
 730         tcg_gen_or_i32(ret, ret, t0);
 731         tcg_temp_free_i32(t0);
 732     }
 733 }
 734
 735 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
 736 {
 737     if (TCG_TARGET_HAS_bswap32_i32) {
 738         tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
 739     } else {
 740         TCGv_i32 t0, t1;
 741         t0 = tcg_temp_new_i32();
 742         t1 = tcg_temp_new_i32();
 743
 744         tcg_gen_shli_i32(t0, arg, 24);
 745
 746         tcg_gen_andi_i32(t1, arg, 0x0000ff00);
 747         tcg_gen_shli_i32(t1, t1, 8);
 748         tcg_gen_or_i32(t0, t0, t1);
 749
 750         tcg_gen_shri_i32(t1, arg, 8);
 751         tcg_gen_andi_i32(t1, t1, 0x0000ff00);
 752         tcg_gen_or_i32(t0, t0, t1);
 753
 754         tcg_gen_shri_i32(t1, arg, 24);
 755         tcg_gen_or_i32(ret, t0, t1);
 756         tcg_temp_free_i32(t0);
 757         tcg_temp_free_i32(t1);
 758     }
 759 }
 760
 761 /* 64-bit ops */
 762
 763 #if TCG_TARGET_REG_BITS == 32
 764 /* These are all inline for TCG_TARGET_REG_BITS == 64.  */
 765
 766 void tcg_gen_discard_i64(TCGv_i64 arg)
 767 {
 768     tcg_gen_discard_i32(TCGV_LOW(arg));
 769     tcg_gen_discard_i32(TCGV_HIGH(arg));
 770 }
 771
 772 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
 773 {
 774     tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
 775     tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
 776 }
 777
 778 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
 779 {
 780     tcg_gen_movi_i32(TCGV_LOW(ret), arg);
 781     tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
 782 }
 783
 784 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
 785 {
 786     tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
 787     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 788 }
 789
 790 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
 791 {
 792     tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
 793     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), 31);
 794 }
 795
 796 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
 797 {
 798     tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
 799     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 800 }
 801
 802 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
 803 {
 804     tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
 805     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
 806 }
 807
 808 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
 809 {
 810     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
 811     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 812 }
 813
 814 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
 815 {
 816     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
 817     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
 818 }
 819
 820 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
 821 {
 822     /* Since arg2 and ret have different types,
 823        they cannot be the same temporary */
 824 #ifdef HOST_WORDS_BIGENDIAN
 825     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
 826     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
 827 #else
 828     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
 829     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
 830 #endif
 831 }
 832
 833 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
 834 {
 835 #ifdef HOST_WORDS_BIGENDIAN
 836     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
 837     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
 838 #else
 839     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
 840     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
 841 #endif
 842 }
 843
 844 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 845 {
 846     tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
 847     tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
 848 }
 849
 850 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 851 {
 852     tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
 853     tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
 854 }
 855
 856 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 857 {
 858     tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
 859     tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
 860 }
 861
 862 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 863 {
 864     gen_helper_shl_i64(ret, arg1, arg2);
 865 }
 866
 867 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 868 {
 869     gen_helper_shr_i64(ret, arg1, arg2);
 870 }
 871
 872 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 873 {
 874     gen_helper_sar_i64(ret, arg1, arg2);
 875 }
 876
 877 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 878 {
 879     TCGv_i64 t0;
 880     TCGv_i32 t1;
 881
 882     t0 = tcg_temp_new_i64();
 883     t1 = tcg_temp_new_i32();
 884
 885     tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
 886                       TCGV_LOW(arg1), TCGV_LOW(arg2));
 887
 888     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
 889     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
 890     tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
 891     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
 892
 893     tcg_gen_mov_i64(ret, t0);
 894     tcg_temp_free_i64(t0);
 895     tcg_temp_free_i32(t1);
 896 }
 897 #endif /* TCG_TARGET_REG_SIZE == 32 */
 898
 899 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 900 {
 901     /* some cases can be optimized here */
 902     if (arg2 == 0) {
 903         tcg_gen_mov_i64(ret, arg1);
 904     } else {
 905         TCGv_i64 t0 = tcg_const_i64(arg2);
 906         tcg_gen_add_i64(ret, arg1, t0);
 907         tcg_temp_free_i64(t0);
 908     }
 909 }
 910
 911 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
 912 {
 913     if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
 914         /* Don't recurse with tcg_gen_neg_i64.  */
 915         tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
 916     } else {
 917         TCGv_i64 t0 = tcg_const_i64(arg1);
 918         tcg_gen_sub_i64(ret, t0, arg2);
 919         tcg_temp_free_i64(t0);
 920     }
 921 }
 922
 923 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 924 {
 925     /* some cases can be optimized here */
 926     if (arg2 == 0) {
 927         tcg_gen_mov_i64(ret, arg1);
 928     } else {
 929         TCGv_i64 t0 = tcg_const_i64(arg2);
 930         tcg_gen_sub_i64(ret, arg1, t0);
 931         tcg_temp_free_i64(t0);
 932     }
 933 }
 934
 935 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
 936 {
 937     TCGv_i64 t0;
 938
 939     if (TCG_TARGET_REG_BITS == 32) {
 940         tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
 941         tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
 942         return;
 943     }
 944
 945     /* Some cases can be optimized here.  */
 946     switch (arg2) {
 947     case 0:
 948         tcg_gen_movi_i64(ret, 0);
 949         return;
 950     case 0xffffffffffffffffull:
 951         tcg_gen_mov_i64(ret, arg1);
 952         return;
 953     case 0xffull:
 954         /* Don't recurse with tcg_gen_ext8u_i64.  */
 955         if (TCG_TARGET_HAS_ext8u_i64) {
 956             tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
 957             return;
 958         }
 959         break;
 960     case 0xffffu:
 961         if (TCG_TARGET_HAS_ext16u_i64) {
 962             tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
 963             return;
 964         }
 965         break;
 966     case 0xffffffffull:
 967         if (TCG_TARGET_HAS_ext32u_i64) {
 968             tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
 969             return;
 970         }
 971         break;
 972     }
 973     t0 = tcg_const_i64(arg2);
 974     tcg_gen_and_i64(ret, arg1, t0);
 975     tcg_temp_free_i64(t0);
 976 }
 977
 978 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 979 {
 980     if (TCG_TARGET_REG_BITS == 32) {
 981         tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
 982         tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
 983         return;
 984     }
 985     /* Some cases can be optimized here.  */
 986     if (arg2 == -1) {
 987         tcg_gen_movi_i64(ret, -1);
 988     } else if (arg2 == 0) {
 989         tcg_gen_mov_i64(ret, arg1);
 990     } else {
 991         TCGv_i64 t0 = tcg_const_i64(arg2);
 992         tcg_gen_or_i64(ret, arg1, t0);
 993         tcg_temp_free_i64(t0);
 994     }
 995 }
 996
 997 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 998 {
 999     if (TCG_TARGET_REG_BITS == 32) {
1000         tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1001         tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1002         return;
1003     }
1004     /* Some cases can be optimized here.  */
1005     if (arg2 == 0) {
1006         tcg_gen_mov_i64(ret, arg1);
1007     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1008         /* Don't recurse with tcg_gen_not_i64.  */
1009         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1010     } else {
1011         TCGv_i64 t0 = tcg_const_i64(arg2);
1012         tcg_gen_xor_i64(ret, arg1, t0);
1013         tcg_temp_free_i64(t0);
1014     }
1015 }
1016
1017 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1018                                       unsigned c, bool right, bool arith)
1019 {
1020     tcg_debug_assert(c < 64);
1021     if (c == 0) {
1022         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1023         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1024     } else if (c >= 32) {
1025         c -= 32;
1026         if (right) {
1027             if (arith) {
1028                 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1029                 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1030             } else {
1031                 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1032                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1033             }
1034         } else {
1035             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1036             tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1037         }
1038     } else {
1039         TCGv_i32 t0, t1;
1040
1041         t0 = tcg_temp_new_i32();
1042         t1 = tcg_temp_new_i32();
1043         if (right) {
1044             tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1045             if (arith) {
1046                 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1047             } else {
1048                 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1049             }
1050             tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1051             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1052             tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1053         } else {
1054             tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1055             /* Note: ret can be the same as arg1, so we use t1 */
1056             tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1057             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1058             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1059             tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1060         }
1061         tcg_temp_free_i32(t0);
1062         tcg_temp_free_i32(t1);
1063     }
1064 }
1065
1066 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1067 {
1068     tcg_debug_assert(arg2 < 64);
1069     if (TCG_TARGET_REG_BITS == 32) {
1070         tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1071     } else if (arg2 == 0) {
1072         tcg_gen_mov_i64(ret, arg1);
1073     } else {
1074         TCGv_i64 t0 = tcg_const_i64(arg2);
1075         tcg_gen_shl_i64(ret, arg1, t0);
1076         tcg_temp_free_i64(t0);
1077     }
1078 }
1079
1080 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1081 {
1082     tcg_debug_assert(arg2 < 64);
1083     if (TCG_TARGET_REG_BITS == 32) {
1084         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1085     } else if (arg2 == 0) {
1086         tcg_gen_mov_i64(ret, arg1);
1087     } else {
1088         TCGv_i64 t0 = tcg_const_i64(arg2);
1089         tcg_gen_shr_i64(ret, arg1, t0);
1090         tcg_temp_free_i64(t0);
1091     }
1092 }
1093
1094 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1095 {
1096     tcg_debug_assert(arg2 < 64);
1097     if (TCG_TARGET_REG_BITS == 32) {
1098         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1099     } else if (arg2 == 0) {
1100         tcg_gen_mov_i64(ret, arg1);
1101     } else {
1102         TCGv_i64 t0 = tcg_const_i64(arg2);
1103         tcg_gen_sar_i64(ret, arg1, t0);
1104         tcg_temp_free_i64(t0);
1105     }
1106 }
1107
1108 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1109 {
1110     if (cond == TCG_COND_ALWAYS) {
1111         tcg_gen_br(l);
1112     } else if (cond != TCG_COND_NEVER) {
1113         if (TCG_TARGET_REG_BITS == 32) {
1114             tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1115                               TCGV_HIGH(arg1), TCGV_LOW(arg2),
1116                               TCGV_HIGH(arg2), cond, label_arg(l));
1117         } else {
1118             tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1119                               label_arg(l));
1120         }
1121     }
1122 }
1123
1124 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1125 {
1126     if (cond == TCG_COND_ALWAYS) {
1127         tcg_gen_br(l);
1128     } else if (cond != TCG_COND_NEVER) {
1129         TCGv_i64 t0 = tcg_const_i64(arg2);
1130         tcg_gen_brcond_i64(cond, arg1, t0, l);
1131         tcg_temp_free_i64(t0);
1132     }
1133 }
1134
1135 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1136                          TCGv_i64 arg1, TCGv_i64 arg2)
1137 {
1138     if (cond == TCG_COND_ALWAYS) {
1139         tcg_gen_movi_i64(ret, 1);
1140     } else if (cond == TCG_COND_NEVER) {
1141         tcg_gen_movi_i64(ret, 0);
1142     } else {
1143         if (TCG_TARGET_REG_BITS == 32) {
1144             tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1145                              TCGV_LOW(arg1), TCGV_HIGH(arg1),
1146                              TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1147             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1148         } else {
1149             tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1150         }
1151     }
1152 }
1153
1154 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1155                           TCGv_i64 arg1, int64_t arg2)
1156 {
1157     TCGv_i64 t0 = tcg_const_i64(arg2);
1158     tcg_gen_setcond_i64(cond, ret, arg1, t0);
1159     tcg_temp_free_i64(t0);
1160 }
1161
1162 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1163 {
1164     TCGv_i64 t0 = tcg_const_i64(arg2);
1165     tcg_gen_mul_i64(ret, arg1, t0);
1166     tcg_temp_free_i64(t0);
1167 }
1168
1169 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1170 {
1171     if (TCG_TARGET_HAS_div_i64) {
1172         tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1173     } else if (TCG_TARGET_HAS_div2_i64) {
1174         TCGv_i64 t0 = tcg_temp_new_i64();
1175         tcg_gen_sari_i64(t0, arg1, 63);
1176         tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1177         tcg_temp_free_i64(t0);
1178     } else {
1179         gen_helper_div_i64(ret, arg1, arg2);
1180     }
1181 }
1182
1183 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1184 {
1185     if (TCG_TARGET_HAS_rem_i64) {
1186         tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1187     } else if (TCG_TARGET_HAS_div_i64) {
1188         TCGv_i64 t0 = tcg_temp_new_i64();
1189         tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1190         tcg_gen_mul_i64(t0, t0, arg2);
1191         tcg_gen_sub_i64(ret, arg1, t0);
1192         tcg_temp_free_i64(t0);
1193     } else if (TCG_TARGET_HAS_div2_i64) {
1194         TCGv_i64 t0 = tcg_temp_new_i64();
1195         tcg_gen_sari_i64(t0, arg1, 63);
1196         tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1197         tcg_temp_free_i64(t0);
1198     } else {
1199         gen_helper_rem_i64(ret, arg1, arg2);
1200     }
1201 }
1202
1203 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1204 {
1205     if (TCG_TARGET_HAS_div_i64) {
1206         tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1207     } else if (TCG_TARGET_HAS_div2_i64) {
1208         TCGv_i64 t0 = tcg_temp_new_i64();
1209         tcg_gen_movi_i64(t0, 0);
1210         tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1211         tcg_temp_free_i64(t0);
1212     } else {
1213         gen_helper_divu_i64(ret, arg1, arg2);
1214     }
1215 }
1216
1217 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1218 {
1219     if (TCG_TARGET_HAS_rem_i64) {
1220         tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1221     } else if (TCG_TARGET_HAS_div_i64) {
1222         TCGv_i64 t0 = tcg_temp_new_i64();
1223         tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1224         tcg_gen_mul_i64(t0, t0, arg2);
1225         tcg_gen_sub_i64(ret, arg1, t0);
1226         tcg_temp_free_i64(t0);
1227     } else if (TCG_TARGET_HAS_div2_i64) {
1228         TCGv_i64 t0 = tcg_temp_new_i64();
1229         tcg_gen_movi_i64(t0, 0);
1230         tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1231         tcg_temp_free_i64(t0);
1232     } else {
1233         gen_helper_remu_i64(ret, arg1, arg2);
1234     }
1235 }
1236
1237 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1238 {
1239     if (TCG_TARGET_REG_BITS == 32) {
1240         tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1241         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1242     } else if (TCG_TARGET_HAS_ext8s_i64) {
1243         tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1244     } else {
1245         tcg_gen_shli_i64(ret, arg, 56);
1246         tcg_gen_sari_i64(ret, ret, 56);
1247     }
1248 }
1249
1250 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1251 {
1252     if (TCG_TARGET_REG_BITS == 32) {
1253         tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1254         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1255     } else if (TCG_TARGET_HAS_ext16s_i64) {
1256         tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1257     } else {
1258         tcg_gen_shli_i64(ret, arg, 48);
1259         tcg_gen_sari_i64(ret, ret, 48);
1260     }
1261 }
1262
1263 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1264 {
1265     if (TCG_TARGET_REG_BITS == 32) {
1266         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1267         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1268     } else if (TCG_TARGET_HAS_ext32s_i64) {
1269         tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1270     } else {
1271         tcg_gen_shli_i64(ret, arg, 32);
1272         tcg_gen_sari_i64(ret, ret, 32);
1273     }
1274 }
1275
1276 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1277 {
1278     if (TCG_TARGET_REG_BITS == 32) {
1279         tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1280         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1281     } else if (TCG_TARGET_HAS_ext8u_i64) {
1282         tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1283     } else {
1284         tcg_gen_andi_i64(ret, arg, 0xffu);
1285     }
1286 }
1287
1288 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1289 {
1290     if (TCG_TARGET_REG_BITS == 32) {
1291         tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1292         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1293     } else if (TCG_TARGET_HAS_ext16u_i64) {
1294         tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1295     } else {
1296         tcg_gen_andi_i64(ret, arg, 0xffffu);
1297     }
1298 }
1299
1300 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1301 {
1302     if (TCG_TARGET_REG_BITS == 32) {
1303         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1304         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1305     } else if (TCG_TARGET_HAS_ext32u_i64) {
1306         tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1307     } else {
1308         tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1309     }
1310 }
1311
1312 /* Note: we assume the six high bytes are set to zero */
1313 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1314 {
1315     if (TCG_TARGET_REG_BITS == 32) {
1316         tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1317         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1318     } else if (TCG_TARGET_HAS_bswap16_i64) {
1319         tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1320     } else {
1321         TCGv_i64 t0 = tcg_temp_new_i64();
1322
1323         tcg_gen_ext8u_i64(t0, arg);
1324         tcg_gen_shli_i64(t0, t0, 8);
1325         tcg_gen_shri_i64(ret, arg, 8);
1326         tcg_gen_or_i64(ret, ret, t0);
1327         tcg_temp_free_i64(t0);
1328     }
1329 }
1330
1331 /* Note: we assume the four high bytes are set to zero */
1332 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1333 {
1334     if (TCG_TARGET_REG_BITS == 32) {
1335         tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1336         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1337     } else if (TCG_TARGET_HAS_bswap32_i64) {
1338         tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1339     } else {
1340         TCGv_i64 t0, t1;
1341         t0 = tcg_temp_new_i64();
1342         t1 = tcg_temp_new_i64();
1343
1344         tcg_gen_shli_i64(t0, arg, 24);
1345         tcg_gen_ext32u_i64(t0, t0);
1346
1347         tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1348         tcg_gen_shli_i64(t1, t1, 8);
1349         tcg_gen_or_i64(t0, t0, t1);
1350
1351         tcg_gen_shri_i64(t1, arg, 8);
1352         tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1353         tcg_gen_or_i64(t0, t0, t1);
1354
1355         tcg_gen_shri_i64(t1, arg, 24);
1356         tcg_gen_or_i64(ret, t0, t1);
1357         tcg_temp_free_i64(t0);
1358         tcg_temp_free_i64(t1);
1359     }
1360 }
1361
1362 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1363 {
1364     if (TCG_TARGET_REG_BITS == 32) {
1365         TCGv_i32 t0, t1;
1366         t0 = tcg_temp_new_i32();
1367         t1 = tcg_temp_new_i32();
1368
1369         tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1370         tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1371         tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1372         tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1373         tcg_temp_free_i32(t0);
1374         tcg_temp_free_i32(t1);
1375     } else if (TCG_TARGET_HAS_bswap64_i64) {
1376         tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1377     } else {
1378         TCGv_i64 t0 = tcg_temp_new_i64();
1379         TCGv_i64 t1 = tcg_temp_new_i64();
1380
1381         tcg_gen_shli_i64(t0, arg, 56);
1382
1383         tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1384         tcg_gen_shli_i64(t1, t1, 40);
1385         tcg_gen_or_i64(t0, t0, t1);
1386
1387         tcg_gen_andi_i64(t1, arg, 0x00ff0000);
1388         tcg_gen_shli_i64(t1, t1, 24);
1389         tcg_gen_or_i64(t0, t0, t1);
1390
1391         tcg_gen_andi_i64(t1, arg, 0xff000000);
1392         tcg_gen_shli_i64(t1, t1, 8);
1393         tcg_gen_or_i64(t0, t0, t1);
1394
1395         tcg_gen_shri_i64(t1, arg, 8);
1396         tcg_gen_andi_i64(t1, t1, 0xff000000);
1397         tcg_gen_or_i64(t0, t0, t1);
1398
1399         tcg_gen_shri_i64(t1, arg, 24);
1400         tcg_gen_andi_i64(t1, t1, 0x00ff0000);
1401         tcg_gen_or_i64(t0, t0, t1);
1402
1403         tcg_gen_shri_i64(t1, arg, 40);
1404         tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1405         tcg_gen_or_i64(t0, t0, t1);
1406
1407         tcg_gen_shri_i64(t1, arg, 56);
1408         tcg_gen_or_i64(ret, t0, t1);
1409         tcg_temp_free_i64(t0);
1410         tcg_temp_free_i64(t1);
1411     }
1412 }
1413
1414 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1415 {
1416     if (TCG_TARGET_REG_BITS == 32) {
1417         tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1418         tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1419     } else if (TCG_TARGET_HAS_not_i64) {
1420         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1421     } else {
1422         tcg_gen_xori_i64(ret, arg, -1);
1423     }
1424 }
1425
1426 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1427 {
1428     if (TCG_TARGET_REG_BITS == 32) {
1429         tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1430         tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1431     } else if (TCG_TARGET_HAS_andc_i64) {
1432         tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1433     } else {
1434         TCGv_i64 t0 = tcg_temp_new_i64();
1435         tcg_gen_not_i64(t0, arg2);
1436         tcg_gen_and_i64(ret, arg1, t0);
1437         tcg_temp_free_i64(t0);
1438     }
1439 }
1440
1441 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1442 {
1443     if (TCG_TARGET_REG_BITS == 32) {
1444         tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1445         tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1446     } else if (TCG_TARGET_HAS_eqv_i64) {
1447         tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1448     } else {
1449         tcg_gen_xor_i64(ret, arg1, arg2);
1450         tcg_gen_not_i64(ret, ret);
1451     }
1452 }
1453
1454 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1455 {
1456     if (TCG_TARGET_REG_BITS == 32) {
1457         tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1458         tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1459     } else if (TCG_TARGET_HAS_nand_i64) {
1460         tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1461     } else {
1462         tcg_gen_and_i64(ret, arg1, arg2);
1463         tcg_gen_not_i64(ret, ret);
1464     }
1465 }
1466
1467 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1468 {
1469     if (TCG_TARGET_REG_BITS == 32) {
1470         tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1471         tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1472     } else if (TCG_TARGET_HAS_nor_i64) {
1473         tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1474     } else {
1475         tcg_gen_or_i64(ret, arg1, arg2);
1476         tcg_gen_not_i64(ret, ret);
1477     }
1478 }
1479
1480 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1481 {
1482     if (TCG_TARGET_REG_BITS == 32) {
1483         tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1484         tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1485     } else if (TCG_TARGET_HAS_orc_i64) {
1486         tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1487     } else {
1488         TCGv_i64 t0 = tcg_temp_new_i64();
1489         tcg_gen_not_i64(t0, arg2);
1490         tcg_gen_or_i64(ret, arg1, t0);
1491         tcg_temp_free_i64(t0);
1492     }
1493 }
1494
1495 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1496 {
1497     if (TCG_TARGET_HAS_rot_i64) {
1498         tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1499     } else {
1500         TCGv_i64 t0, t1;
1501         t0 = tcg_temp_new_i64();
1502         t1 = tcg_temp_new_i64();
1503         tcg_gen_shl_i64(t0, arg1, arg2);
1504         tcg_gen_subfi_i64(t1, 64, arg2);
1505         tcg_gen_shr_i64(t1, arg1, t1);
1506         tcg_gen_or_i64(ret, t0, t1);
1507         tcg_temp_free_i64(t0);
1508         tcg_temp_free_i64(t1);
1509     }
1510 }
1511
1512 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1513 {
1514     tcg_debug_assert(arg2 < 64);
1515     /* some cases can be optimized here */
1516     if (arg2 == 0) {
1517         tcg_gen_mov_i64(ret, arg1);
1518     } else if (TCG_TARGET_HAS_rot_i64) {
1519         TCGv_i64 t0 = tcg_const_i64(arg2);
1520         tcg_gen_rotl_i64(ret, arg1, t0);
1521         tcg_temp_free_i64(t0);
1522     } else {
1523         TCGv_i64 t0, t1;
1524         t0 = tcg_temp_new_i64();
1525         t1 = tcg_temp_new_i64();
1526         tcg_gen_shli_i64(t0, arg1, arg2);
1527         tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1528         tcg_gen_or_i64(ret, t0, t1);
1529         tcg_temp_free_i64(t0);
1530         tcg_temp_free_i64(t1);
1531     }
1532 }
1533
1534 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1535 {
1536     if (TCG_TARGET_HAS_rot_i64) {
1537         tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1538     } else {
1539         TCGv_i64 t0, t1;
1540         t0 = tcg_temp_new_i64();
1541         t1 = tcg_temp_new_i64();
1542         tcg_gen_shr_i64(t0, arg1, arg2);
1543         tcg_gen_subfi_i64(t1, 64, arg2);
1544         tcg_gen_shl_i64(t1, arg1, t1);
1545         tcg_gen_or_i64(ret, t0, t1);
1546         tcg_temp_free_i64(t0);
1547         tcg_temp_free_i64(t1);
1548     }
1549 }
1550
1551 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1552 {
1553     tcg_debug_assert(arg2 < 64);
1554     /* some cases can be optimized here */
1555     if (arg2 == 0) {
1556         tcg_gen_mov_i64(ret, arg1);
1557     } else {
1558         tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1559     }
1560 }
1561
1562 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1563                          unsigned int ofs, unsigned int len)
1564 {
1565     uint64_t mask;
1566     TCGv_i64 t1;
1567
1568     tcg_debug_assert(ofs < 64);
1569     tcg_debug_assert(len <= 64);
1570     tcg_debug_assert(ofs + len <= 64);
1571
1572     if (ofs == 0 && len == 64) {
1573         tcg_gen_mov_i64(ret, arg2);
1574         return;
1575     }
1576     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1577         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1578         return;
1579     }
1580
1581     if (TCG_TARGET_REG_BITS == 32) {
1582         if (ofs >= 32) {
1583             tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1584                                 TCGV_LOW(arg2), ofs - 32, len);
1585             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1586             return;
1587         }
1588         if (ofs + len <= 32) {
1589             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
1590                                 TCGV_LOW(arg2), ofs, len);
1591             tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1592             return;
1593         }
1594     }
1595
1596     mask = (1ull << len) - 1;
1597     t1 = tcg_temp_new_i64();
1598
1599     if (ofs + len < 64) {
1600         tcg_gen_andi_i64(t1, arg2, mask);
1601         tcg_gen_shli_i64(t1, t1, ofs);
1602     } else {
1603         tcg_gen_shli_i64(t1, arg2, ofs);
1604     }
1605     tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
1606     tcg_gen_or_i64(ret, ret, t1);
1607
1608     tcg_temp_free_i64(t1);
1609 }
1610
1611 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
1612                          TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
1613 {
1614     if (cond == TCG_COND_ALWAYS) {
1615         tcg_gen_mov_i64(ret, v1);
1616     } else if (cond == TCG_COND_NEVER) {
1617         tcg_gen_mov_i64(ret, v2);
1618     } else if (TCG_TARGET_REG_BITS == 32) {
1619         TCGv_i32 t0 = tcg_temp_new_i32();
1620         TCGv_i32 t1 = tcg_temp_new_i32();
1621         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
1622                          TCGV_LOW(c1), TCGV_HIGH(c1),
1623                          TCGV_LOW(c2), TCGV_HIGH(c2), cond);
1624
1625         if (TCG_TARGET_HAS_movcond_i32) {
1626             tcg_gen_movi_i32(t1, 0);
1627             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
1628                                 TCGV_LOW(v1), TCGV_LOW(v2));
1629             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
1630                                 TCGV_HIGH(v1), TCGV_HIGH(v2));
1631         } else {
1632             tcg_gen_neg_i32(t0, t0);
1633
1634             tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
1635             tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
1636             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
1637
1638             tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
1639             tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
1640             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
1641         }
1642         tcg_temp_free_i32(t0);
1643         tcg_temp_free_i32(t1);
1644     } else if (TCG_TARGET_HAS_movcond_i64) {
1645         tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
1646     } else {
1647         TCGv_i64 t0 = tcg_temp_new_i64();
1648         TCGv_i64 t1 = tcg_temp_new_i64();
1649         tcg_gen_setcond_i64(cond, t0, c1, c2);
1650         tcg_gen_neg_i64(t0, t0);
1651         tcg_gen_and_i64(t1, v1, t0);
1652         tcg_gen_andc_i64(ret, v2, t0);
1653         tcg_gen_or_i64(ret, ret, t1);
1654         tcg_temp_free_i64(t0);
1655         tcg_temp_free_i64(t1);
1656     }
1657 }
1658
1659 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
1660                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
1661 {
1662     if (TCG_TARGET_HAS_add2_i64) {
1663         tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
1664     } else {
1665         TCGv_i64 t0 = tcg_temp_new_i64();
1666         TCGv_i64 t1 = tcg_temp_new_i64();
1667         tcg_gen_add_i64(t0, al, bl);
1668         tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
1669         tcg_gen_add_i64(rh, ah, bh);
1670         tcg_gen_add_i64(rh, rh, t1);
1671         tcg_gen_mov_i64(rl, t0);
1672         tcg_temp_free_i64(t0);
1673         tcg_temp_free_i64(t1);
1674     }
1675 }
1676
1677 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
1678                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
1679 {
1680     if (TCG_TARGET_HAS_sub2_i64) {
1681         tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
1682     } else {
1683         TCGv_i64 t0 = tcg_temp_new_i64();
1684         TCGv_i64 t1 = tcg_temp_new_i64();
1685         tcg_gen_sub_i64(t0, al, bl);
1686         tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
1687         tcg_gen_sub_i64(rh, ah, bh);
1688         tcg_gen_sub_i64(rh, rh, t1);
1689         tcg_gen_mov_i64(rl, t0);
1690         tcg_temp_free_i64(t0);
1691         tcg_temp_free_i64(t1);
1692     }
1693 }
1694
1695 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
1696 {
1697     if (TCG_TARGET_HAS_mulu2_i64) {
1698         tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
1699     } else if (TCG_TARGET_HAS_muluh_i64) {
1700         TCGv_i64 t = tcg_temp_new_i64();
1701         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
1702         tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
1703         tcg_gen_mov_i64(rl, t);
1704         tcg_temp_free_i64(t);
1705     } else {
1706         TCGv_i64 t0 = tcg_temp_new_i64();
1707         tcg_gen_mul_i64(t0, arg1, arg2);
1708         gen_helper_muluh_i64(rh, arg1, arg2);
1709         tcg_gen_mov_i64(rl, t0);
1710         tcg_temp_free_i64(t0);
1711     }
1712 }
1713
1714 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
1715 {
1716     if (TCG_TARGET_HAS_muls2_i64) {
1717         tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
1718     } else if (TCG_TARGET_HAS_mulsh_i64) {
1719         TCGv_i64 t = tcg_temp_new_i64();
1720         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
1721         tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
1722         tcg_gen_mov_i64(rl, t);
1723         tcg_temp_free_i64(t);
1724     } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
1725         TCGv_i64 t0 = tcg_temp_new_i64();
1726         TCGv_i64 t1 = tcg_temp_new_i64();
1727         TCGv_i64 t2 = tcg_temp_new_i64();
1728         TCGv_i64 t3 = tcg_temp_new_i64();
1729         tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
1730         /* Adjust for negative inputs.  */
1731         tcg_gen_sari_i64(t2, arg1, 63);
1732         tcg_gen_sari_i64(t3, arg2, 63);
1733         tcg_gen_and_i64(t2, t2, arg2);
1734         tcg_gen_and_i64(t3, t3, arg1);
1735         tcg_gen_sub_i64(rh, t1, t2);
1736         tcg_gen_sub_i64(rh, rh, t3);
1737         tcg_gen_mov_i64(rl, t0);
1738         tcg_temp_free_i64(t0);
1739         tcg_temp_free_i64(t1);
1740         tcg_temp_free_i64(t2);
1741         tcg_temp_free_i64(t3);
1742     } else {
1743         TCGv_i64 t0 = tcg_temp_new_i64();
1744         tcg_gen_mul_i64(t0, arg1, arg2);
1745         gen_helper_mulsh_i64(rh, arg1, arg2);
1746         tcg_gen_mov_i64(rl, t0);
1747         tcg_temp_free_i64(t0);
1748     }
1749 }
1750
1751 /* Size changing operations.  */
1752
1753 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
1754 {
1755     if (TCG_TARGET_REG_BITS == 32) {
1756         tcg_gen_mov_i32(ret, TCGV_LOW(arg));
1757     } else if (TCG_TARGET_HAS_extrl_i64_i32) {
1758         tcg_gen_op2(&tcg_ctx, INDEX_op_extrl_i64_i32,
1759                     GET_TCGV_I32(ret), GET_TCGV_I64(arg));
1760     } else {
1761         tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(arg)));
1762     }
1763 }
1764
1765 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
1766 {
1767     if (TCG_TARGET_REG_BITS == 32) {
1768         tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
1769     } else if (TCG_TARGET_HAS_extrh_i64_i32) {
1770         tcg_gen_op2(&tcg_ctx, INDEX_op_extrh_i64_i32,
1771                     GET_TCGV_I32(ret), GET_TCGV_I64(arg));
1772     } else {
1773         TCGv_i64 t = tcg_temp_new_i64();
1774         tcg_gen_shri_i64(t, arg, 32);
1775         tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(t)));
1776         tcg_temp_free_i64(t);
1777     }
1778 }
1779
1780 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
1781 {
1782     if (TCG_TARGET_REG_BITS == 32) {
1783         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
1784         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1785     } else {
1786         tcg_gen_op2(&tcg_ctx, INDEX_op_extu_i32_i64,
1787                     GET_TCGV_I64(ret), GET_TCGV_I32(arg));
1788     }
1789 }
1790
1791 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
1792 {
1793     if (TCG_TARGET_REG_BITS == 32) {
1794         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
1795         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1796     } else {
1797         tcg_gen_op2(&tcg_ctx, INDEX_op_ext_i32_i64,
1798                     GET_TCGV_I64(ret), GET_TCGV_I32(arg));
1799     }
1800 }
1801
1802 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
1803 {
1804     TCGv_i64 tmp;
1805
1806     if (TCG_TARGET_REG_BITS == 32) {
1807         tcg_gen_mov_i32(TCGV_LOW(dest), low);
1808         tcg_gen_mov_i32(TCGV_HIGH(dest), high);
1809         return;
1810     }
1811
1812     tmp = tcg_temp_new_i64();
1813     /* These extensions are only needed for type correctness.
1814        We may be able to do better given target specific information.  */
1815     tcg_gen_extu_i32_i64(tmp, high);
1816     tcg_gen_extu_i32_i64(dest, low);
1817     /* If deposit is available, use it.  Otherwise use the extra
1818        knowledge that we have of the zero-extensions above.  */
1819     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
1820         tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
1821     } else {
1822         tcg_gen_shli_i64(tmp, tmp, 32);
1823         tcg_gen_or_i64(dest, dest, tmp);
1824     }
1825     tcg_temp_free_i64(tmp);
1826 }
1827
1828 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
1829 {
1830     if (TCG_TARGET_REG_BITS == 32) {
1831         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
1832         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
1833     } else {
1834         tcg_gen_extrl_i64_i32(lo, arg);
1835         tcg_gen_extrh_i64_i32(hi, arg);
1836     }
1837 }
1838
1839 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
1840 {
1841     tcg_gen_ext32u_i64(lo, arg);
1842     tcg_gen_shri_i64(hi, arg, 32);
1843 }
1844
1845 /* QEMU specific operations.  */
1846
1847 void tcg_gen_goto_tb(unsigned idx)
1848 {
1849     /* We only support two chained exits.  */
1850     tcg_debug_assert(idx <= 1);
1851 #ifdef CONFIG_DEBUG_TCG
1852     /* Verify that we havn't seen this numbered exit before.  */
1853     tcg_debug_assert((tcg_ctx.goto_tb_issue_mask & (1 << idx)) == 0);
1854     tcg_ctx.goto_tb_issue_mask |= 1 << idx;
1855 #endif
1856     tcg_gen_op1i(INDEX_op_goto_tb, idx);
1857 }
1858
1859 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
1860 {
1861     /* Trigger the asserts within as early as possible.  */
1862     (void)get_alignment_bits(op);
1863
1864     switch (op & MO_SIZE) {
1865     case MO_8:
1866         op &= ~MO_BSWAP;
1867         break;
1868     case MO_16:
1869         break;
1870     case MO_32:
1871         if (!is64) {
1872             op &= ~MO_SIGN;
1873         }
1874         break;
1875     case MO_64:
1876         if (!is64) {
1877             tcg_abort();
1878         }
1879         break;
1880     }
1881     if (st) {
1882         op &= ~MO_SIGN;
1883     }
1884     return op;
1885 }
1886
1887 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
1888                          TCGMemOp memop, TCGArg idx)
1889 {
1890     TCGMemOpIdx oi = make_memop_idx(memop, idx);
1891 #if TARGET_LONG_BITS == 32
1892     tcg_gen_op3i_i32(opc, val, addr, oi);
1893 #else
1894     if (TCG_TARGET_REG_BITS == 32) {
1895         tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
1896     } else {
1897         tcg_gen_op3(&tcg_ctx, opc, GET_TCGV_I32(val), GET_TCGV_I64(addr), oi);
1898     }
1899 #endif
1900 }
1901
1902 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
1903                          TCGMemOp memop, TCGArg idx)
1904 {
1905     TCGMemOpIdx oi = make_memop_idx(memop, idx);
1906 #if TARGET_LONG_BITS == 32
1907     if (TCG_TARGET_REG_BITS == 32) {
1908         tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
1909     } else {
1910         tcg_gen_op3(&tcg_ctx, opc, GET_TCGV_I64(val), GET_TCGV_I32(addr), oi);
1911     }
1912 #else
1913     if (TCG_TARGET_REG_BITS == 32) {
1914         tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
1915                          TCGV_LOW(addr), TCGV_HIGH(addr), oi);
1916     } else {
1917         tcg_gen_op3i_i64(opc, val, addr, oi);
1918     }
1919 #endif
1920 }
1921
1922 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
1923 {
1924     memop = tcg_canonicalize_memop(memop, 0, 0);
1925     trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
1926                                addr, trace_mem_get_info(memop, 0));
1927     gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
1928 }
1929
1930 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
1931 {
1932     memop = tcg_canonicalize_memop(memop, 0, 1);
1933     trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
1934                                addr, trace_mem_get_info(memop, 1));
1935     gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
1936 }
1937
1938 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
1939 {
1940     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
1941         tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
1942         if (memop & MO_SIGN) {
1943             tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
1944         } else {
1945             tcg_gen_movi_i32(TCGV_HIGH(val), 0);
1946         }
1947         return;
1948     }
1949
1950     memop = tcg_canonicalize_memop(memop, 1, 0);
1951     trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
1952                                addr, trace_mem_get_info(memop, 0));
1953     gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
1954 }
1955
1956 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
1957 {
1958     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
1959         tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
1960         return;
1961     }
1962
1963     memop = tcg_canonicalize_memop(memop, 1, 1);
1964     trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
1965                                addr, trace_mem_get_info(memop, 1));
1966     gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
1967 }
1968
1969 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
1970 {
1971     switch (opc & MO_SSIZE) {
1972     case MO_SB:
1973         tcg_gen_ext8s_i32(ret, val);
1974         break;
1975     case MO_UB:
1976         tcg_gen_ext8u_i32(ret, val);
1977         break;
1978     case MO_SW:
1979         tcg_gen_ext16s_i32(ret, val);
1980         break;
1981     case MO_UW:
1982         tcg_gen_ext16u_i32(ret, val);
1983         break;
1984     default:
1985         tcg_gen_mov_i32(ret, val);
1986         break;
1987     }
1988 }
1989
1990 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
1991 {
1992     switch (opc & MO_SSIZE) {
1993     case MO_SB:
1994         tcg_gen_ext8s_i64(ret, val);
1995         break;
1996     case MO_UB:
1997         tcg_gen_ext8u_i64(ret, val);
1998         break;
1999     case MO_SW:
2000         tcg_gen_ext16s_i64(ret, val);
2001         break;
2002     case MO_UW:
2003         tcg_gen_ext16u_i64(ret, val);
2004         break;
2005     case MO_SL:
2006         tcg_gen_ext32s_i64(ret, val);
2007         break;
2008     case MO_UL:
2009         tcg_gen_ext32u_i64(ret, val);
2010         break;
2011     default:
2012         tcg_gen_mov_i64(ret, val);
2013         break;
2014     }
2015 }
2016
2017 #ifdef CONFIG_SOFTMMU
2018 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2019                                   TCGv_i32, TCGv_i32, TCGv_i32);
2020 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2021                                   TCGv_i64, TCGv_i64, TCGv_i32);
2022 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2023                                   TCGv_i32, TCGv_i32);
2024 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2025                                   TCGv_i64, TCGv_i32);
2026 #else
2027 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2028 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2029 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2030 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2031 #endif
2032
2033 #ifdef CONFIG_ATOMIC64
2034 # define WITH_ATOMIC64(X) X,
2035 #else
2036 # define WITH_ATOMIC64(X)
2037 #endif
2038
2039 static void * const table_cmpxchg[16] = {
2040     [MO_8] = gen_helper_atomic_cmpxchgb,
2041     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2042     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2043     [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2044     [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
2045     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2046     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
2047 };
2048
2049 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2050                                 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2051 {
2052     memop = tcg_canonicalize_memop(memop, 0, 0);
2053
2054     if (!parallel_cpus) {
2055         TCGv_i32 t1 = tcg_temp_new_i32();
2056         TCGv_i32 t2 = tcg_temp_new_i32();
2057
2058         tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2059
2060         tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2061         tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2062         tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2063         tcg_temp_free_i32(t2);
2064
2065         if (memop & MO_SIGN) {
2066             tcg_gen_ext_i32(retv, t1, memop);
2067         } else {
2068             tcg_gen_mov_i32(retv, t1);
2069         }
2070         tcg_temp_free_i32(t1);
2071     } else {
2072         gen_atomic_cx_i32 gen;
2073
2074         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2075         tcg_debug_assert(gen != NULL);
2076
2077 #ifdef CONFIG_SOFTMMU
2078         {
2079             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2080             gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi);
2081             tcg_temp_free_i32(oi);
2082         }
2083 #else
2084         gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv);
2085 #endif
2086
2087         if (memop & MO_SIGN) {
2088             tcg_gen_ext_i32(retv, retv, memop);
2089         }
2090     }
2091 }
2092
2093 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2094                                 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2095 {
2096     memop = tcg_canonicalize_memop(memop, 1, 0);
2097
2098     if (!parallel_cpus) {
2099         TCGv_i64 t1 = tcg_temp_new_i64();
2100         TCGv_i64 t2 = tcg_temp_new_i64();
2101
2102         tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2103
2104         tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2105         tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2106         tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2107         tcg_temp_free_i64(t2);
2108
2109         if (memop & MO_SIGN) {
2110             tcg_gen_ext_i64(retv, t1, memop);
2111         } else {
2112             tcg_gen_mov_i64(retv, t1);
2113         }
2114         tcg_temp_free_i64(t1);
2115     } else if ((memop & MO_SIZE) == MO_64) {
2116 #ifdef CONFIG_ATOMIC64
2117         gen_atomic_cx_i64 gen;
2118
2119         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2120         tcg_debug_assert(gen != NULL);
2121
2122 #ifdef CONFIG_SOFTMMU
2123         {
2124             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
2125             gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi);
2126             tcg_temp_free_i32(oi);
2127         }
2128 #else
2129         gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv);
2130 #endif
2131 #else
2132         gen_helper_exit_atomic(tcg_ctx.tcg_env);
2133 #endif /* CONFIG_ATOMIC64 */
2134     } else {
2135         TCGv_i32 c32 = tcg_temp_new_i32();
2136         TCGv_i32 n32 = tcg_temp_new_i32();
2137         TCGv_i32 r32 = tcg_temp_new_i32();
2138
2139         tcg_gen_extrl_i64_i32(c32, cmpv);
2140         tcg_gen_extrl_i64_i32(n32, newv);
2141         tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
2142         tcg_temp_free_i32(c32);
2143         tcg_temp_free_i32(n32);
2144
2145         tcg_gen_extu_i32_i64(retv, r32);
2146         tcg_temp_free_i32(r32);
2147
2148         if (memop & MO_SIGN) {
2149             tcg_gen_ext_i64(retv, retv, memop);
2150         }
2151     }
2152 }
2153
2154 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2155                                 TCGArg idx, TCGMemOp memop, bool new_val,
2156                                 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
2157 {
2158     TCGv_i32 t1 = tcg_temp_new_i32();
2159     TCGv_i32 t2 = tcg_temp_new_i32();
2160
2161     memop = tcg_canonicalize_memop(memop, 0, 0);
2162
2163     tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2164     gen(t2, t1, val);
2165     tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2166
2167     tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
2168     tcg_temp_free_i32(t1);
2169     tcg_temp_free_i32(t2);
2170 }
2171
2172 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2173                              TCGArg idx, TCGMemOp memop, void * const table[])
2174 {
2175     gen_atomic_op_i32 gen;
2176
2177     memop = tcg_canonicalize_memop(memop, 0, 0);
2178
2179     gen = table[memop & (MO_SIZE | MO_BSWAP)];
2180     tcg_debug_assert(gen != NULL);
2181
2182 #ifdef CONFIG_SOFTMMU
2183     {
2184         TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2185         gen(ret, tcg_ctx.tcg_env, addr, val, oi);
2186         tcg_temp_free_i32(oi);
2187     }
2188 #else
2189     gen(ret, tcg_ctx.tcg_env, addr, val);
2190 #endif
2191
2192     if (memop & MO_SIGN) {
2193         tcg_gen_ext_i32(ret, ret, memop);
2194     }
2195 }
2196
2197 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2198                                 TCGArg idx, TCGMemOp memop, bool new_val,
2199                                 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
2200 {
2201     TCGv_i64 t1 = tcg_temp_new_i64();
2202     TCGv_i64 t2 = tcg_temp_new_i64();
2203
2204     memop = tcg_canonicalize_memop(memop, 1, 0);
2205
2206     tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2207     gen(t2, t1, val);
2208     tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2209
2210     tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
2211     tcg_temp_free_i64(t1);
2212     tcg_temp_free_i64(t2);
2213 }
2214
2215 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2216                              TCGArg idx, TCGMemOp memop, void * const table[])
2217 {
2218     memop = tcg_canonicalize_memop(memop, 1, 0);
2219
2220     if ((memop & MO_SIZE) == MO_64) {
2221 #ifdef CONFIG_ATOMIC64
2222         gen_atomic_op_i64 gen;
2223
2224         gen = table[memop & (MO_SIZE | MO_BSWAP)];
2225         tcg_debug_assert(gen != NULL);
2226
2227 #ifdef CONFIG_SOFTMMU
2228         {
2229             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2230             gen(ret, tcg_ctx.tcg_env, addr, val, oi);
2231             tcg_temp_free_i32(oi);
2232         }
2233 #else
2234         gen(ret, tcg_ctx.tcg_env, addr, val);
2235 #endif
2236 #else
2237         gen_helper_exit_atomic(tcg_ctx.tcg_env);
2238 #endif /* CONFIG_ATOMIC64 */
2239     } else {
2240         TCGv_i32 v32 = tcg_temp_new_i32();
2241         TCGv_i32 r32 = tcg_temp_new_i32();
2242
2243         tcg_gen_extrl_i64_i32(v32, val);
2244         do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
2245         tcg_temp_free_i32(v32);
2246
2247         tcg_gen_extu_i32_i64(ret, r32);
2248         tcg_temp_free_i32(r32);
2249
2250         if (memop & MO_SIGN) {
2251             tcg_gen_ext_i64(ret, ret, memop);
2252         }
2253     }
2254 }
2255
2256 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
2257 static void * const table_##NAME[16] = {                                \
2258     [MO_8] = gen_helper_atomic_##NAME##b,                               \
2259     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
2260     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
2261     [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
2262     [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
2263     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le)     \
2264     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be)     \
2265 };                                                                      \
2266 void tcg_gen_atomic_##NAME##_i32                                        \
2267     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
2268 {                                                                       \
2269     if (parallel_cpus) {                                                \
2270         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
2271     } else {                                                            \
2272         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
2273                             tcg_gen_##OP##_i32);                        \
2274     }                                                                   \
2275 }                                                                       \
2276 void tcg_gen_atomic_##NAME##_i64                                        \
2277     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
2278 {                                                                       \
2279     if (parallel_cpus) {                                                \
2280         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
2281     } else {                                                            \
2282         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
2283                             tcg_gen_##OP##_i64);                        \
2284     }                                                                   \
2285 }
2286
2287 GEN_ATOMIC_HELPER(fetch_add, add, 0)
2288 GEN_ATOMIC_HELPER(fetch_and, and, 0)
2289 GEN_ATOMIC_HELPER(fetch_or, or, 0)
2290 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
2291
2292 GEN_ATOMIC_HELPER(add_fetch, add, 1)
2293 GEN_ATOMIC_HELPER(and_fetch, and, 1)
2294 GEN_ATOMIC_HELPER(or_fetch, or, 1)
2295 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
2296
2297 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
2298 {
2299     tcg_gen_mov_i32(r, b);
2300 }
2301
2302 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
2303 {
2304     tcg_gen_mov_i64(r, b);
2305 }
2306
2307 GEN_ATOMIC_HELPER(xchg, mov2, 0)
2308
2309 #undef GEN_ATOMIC_HELPER