src/lj_ffrecord.c

   1 /*
   2 ** Fast function call recorder.
   3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_ffrecord_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10
  11 #if LJ_HASJIT
  12
  13 #include "lj_err.h"
  14 #include "lj_str.h"
  15 #include "lj_tab.h"
  16 #include "lj_frame.h"
  17 #include "lj_bc.h"
  18 #include "lj_ff.h"
  19 #include "lj_ir.h"
  20 #include "lj_jit.h"
  21 #include "lj_ircall.h"
  22 #include "lj_iropt.h"
  23 #include "lj_trace.h"
  24 #include "lj_record.h"
  25 #include "lj_ffrecord.h"
  26 #include "lj_crecord.h"
  27 #include "lj_dispatch.h"
  28 #include "lj_vm.h"
  29 #include "lj_strscan.h"
  30
  31 /* Some local macros to save typing. Undef'd at the end. */
  32 #define IR(ref)                 (&J->cur.ir[(ref)])
  33
  34 /* Pass IR on to next optimization in chain (FOLD). */
  35 #define emitir(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
  36
  37 /* -- Fast function recording handlers ------------------------------------ */
  38
  39 /* Conventions for fast function call handlers:
  40 **
  41 ** The argument slots start at J->base[0]. All of them are guaranteed to be
  42 ** valid and type-specialized references. J->base[J->maxslot] is set to 0
  43 ** as a sentinel. The runtime argument values start at rd->argv[0].
  44 **
  45 ** In general fast functions should check for presence of all of their
  46 ** arguments and for the correct argument types. Some simplifications
  47 ** are allowed if the interpreter throws instead. But even if recording
  48 ** is aborted, the generated IR must be consistent (no zero-refs).
  49 **
  50 ** The number of results in rd->nres is set to 1. Handlers that return
  51 ** a different number of results need to override it. A negative value
  52 ** prevents return processing (e.g. for pending calls).
  53 **
  54 ** Results need to be stored starting at J->base[0]. Return processing
  55 ** moves them to the right slots later.
  56 **
  57 ** The per-ffid auxiliary data is the value of the 2nd part of the
  58 ** LJLIB_REC() annotation. This allows handling similar functionality
  59 ** in a common handler.
  60 */
  61
  62 /* Type of handler to record a fast function. */
  63 typedef void (LJ_FASTCALL *RecordFunc)(jit_State *J, RecordFFData *rd);
  64
  65 /* Get runtime value of int argument. */
  66 static int32_t argv2int(jit_State *J, TValue *o)
  67 {
  68   if (!lj_strscan_numberobj(o))
  69     lj_trace_err(J, LJ_TRERR_BADTYPE);
  70   return tvisint(o) ? intV(o) : lj_num2int(numV(o));
  71 }
  72
  73 /* Get runtime value of string argument. */
  74 static GCstr *argv2str(jit_State *J, TValue *o)
  75 {
  76   if (LJ_LIKELY(tvisstr(o))) {
  77     return strV(o);
  78   } else {
  79     GCstr *s;
  80     if (!tvisnumber(o))
  81       lj_trace_err(J, LJ_TRERR_BADTYPE);
  82     if (tvisint(o))
  83       s = lj_str_fromint(J->L, intV(o));
  84     else
  85       s = lj_str_fromnum(J->L, &o->n);
  86     setstrV(J->L, o, s);
  87     return s;
  88   }
  89 }
  90
  91 /* Return number of results wanted by caller. */
  92 static ptrdiff_t results_wanted(jit_State *J)
  93 {
  94   TValue *frame = J->L->base-1;
  95   if (frame_islua(frame))
  96     return (ptrdiff_t)bc_b(frame_pc(frame)[-1]) - 1;
  97   else
  98     return -1;
  99 }
 100
 101 /* Throw error for unsupported variant of fast function. */
 102 LJ_NORET static void recff_nyiu(jit_State *J)
 103 {
 104   setfuncV(J->L, &J->errinfo, J->fn);
 105   lj_trace_err_info(J, LJ_TRERR_NYIFFU);
 106 }
 107
 108 /* Fallback handler for all fast functions that are not recorded (yet). */
 109 static void LJ_FASTCALL recff_nyi(jit_State *J, RecordFFData *rd)
 110 {
 111   setfuncV(J->L, &J->errinfo, J->fn);
 112   lj_trace_err_info(J, LJ_TRERR_NYIFF);
 113   UNUSED(rd);
 114 }
 115
 116 /* C functions can have arbitrary side-effects and are not recorded (yet). */
 117 static void LJ_FASTCALL recff_c(jit_State *J, RecordFFData *rd)
 118 {
 119   setfuncV(J->L, &J->errinfo, J->fn);
 120   lj_trace_err_info(J, LJ_TRERR_NYICF);
 121   UNUSED(rd);
 122 }
 123
 124 /* -- Base library fast functions ----------------------------------------- */
 125
 126 static void LJ_FASTCALL recff_assert(jit_State *J, RecordFFData *rd)
 127 {
 128   /* Arguments already specialized. The interpreter throws for nil/false. */
 129   rd->nres = J->maxslot;  /* Pass through all arguments. */
 130 }
 131
 132 static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd)
 133 {
 134   /* Arguments already specialized. Result is a constant string. Neat, huh? */
 135   uint32_t t;
 136   if (tvisnumber(&rd->argv[0]))
 137     t = ~LJ_TNUMX;
 138   else if (LJ_64 && tvislightud(&rd->argv[0]))
 139     t = ~LJ_TLIGHTUD;
 140   else
 141     t = ~itype(&rd->argv[0]);
 142   J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[t]));
 143   UNUSED(rd);
 144 }
 145
 146 static void LJ_FASTCALL recff_getmetatable(jit_State *J, RecordFFData *rd)
 147 {
 148   TRef tr = J->base[0];
 149   if (tr) {
 150     RecordIndex ix;
 151     ix.tab = tr;
 152     copyTV(J->L, &ix.tabv, &rd->argv[0]);
 153     if (lj_record_mm_lookup(J, &ix, MM_metatable))
 154       J->base[0] = ix.mobj;
 155     else
 156       J->base[0] = ix.mt;
 157   }  /* else: Interpreter will throw. */
 158 }
 159
 160 static void LJ_FASTCALL recff_setmetatable(jit_State *J, RecordFFData *rd)
 161 {
 162   TRef tr = J->base[0];
 163   TRef mt = J->base[1];
 164   if (tref_istab(tr) && (tref_istab(mt) || (mt && tref_isnil(mt)))) {
 165     TRef fref, mtref;
 166     RecordIndex ix;
 167     ix.tab = tr;
 168     copyTV(J->L, &ix.tabv, &rd->argv[0]);
 169     lj_record_mm_lookup(J, &ix, MM_metatable); /* Guard for no __metatable. */
 170     fref = emitir(IRT(IR_FREF, IRT_P32), tr, IRFL_TAB_META);
 171     mtref = tref_isnil(mt) ? lj_ir_knull(J, IRT_TAB) : mt;
 172     emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref);
 173     if (!tref_isnil(mt))
 174       emitir(IRT(IR_TBAR, IRT_TAB), tr, 0);
 175     J->base[0] = tr;
 176     J->needsnap = 1;
 177   }  /* else: Interpreter will throw. */
 178 }
 179
 180 static void LJ_FASTCALL recff_rawget(jit_State *J, RecordFFData *rd)
 181 {
 182   RecordIndex ix;
 183   ix.tab = J->base[0]; ix.key = J->base[1];
 184   if (tref_istab(ix.tab) && ix.key) {
 185     ix.val = 0; ix.idxchain = 0;
 186     settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
 187     copyTV(J->L, &ix.keyv, &rd->argv[1]);
 188     J->base[0] = lj_record_idx(J, &ix);
 189   }  /* else: Interpreter will throw. */
 190 }
 191
 192 static void LJ_FASTCALL recff_rawset(jit_State *J, RecordFFData *rd)
 193 {
 194   RecordIndex ix;
 195   ix.tab = J->base[0]; ix.key = J->base[1]; ix.val = J->base[2];
 196   if (tref_istab(ix.tab) && ix.key && ix.val) {
 197     ix.idxchain = 0;
 198     settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
 199     copyTV(J->L, &ix.keyv, &rd->argv[1]);
 200     copyTV(J->L, &ix.valv, &rd->argv[2]);
 201     lj_record_idx(J, &ix);
 202     /* Pass through table at J->base[0] as result. */
 203   }  /* else: Interpreter will throw. */
 204 }
 205
 206 static void LJ_FASTCALL recff_rawequal(jit_State *J, RecordFFData *rd)
 207 {
 208   TRef tra = J->base[0];
 209   TRef trb = J->base[1];
 210   if (tra && trb) {
 211     int diff = lj_record_objcmp(J, tra, trb, &rd->argv[0], &rd->argv[1]);
 212     J->base[0] = diff ? TREF_FALSE : TREF_TRUE;
 213   }  /* else: Interpreter will throw. */
 214 }
 215
 216 #if LJ_52
 217 static void LJ_FASTCALL recff_rawlen(jit_State *J, RecordFFData *rd)
 218 {
 219   TRef tr = J->base[0];
 220   if (tref_isstr(tr))
 221     J->base[0] = emitir(IRTI(IR_FLOAD), tr, IRFL_STR_LEN);
 222   else if (tref_istab(tr))
 223     J->base[0] = lj_ir_call(J, IRCALL_lj_tab_len, tr);
 224   /* else: Interpreter will throw. */
 225   UNUSED(rd);
 226 }
 227 #endif
 228
 229 /* Determine mode of select() call. */
 230 int32_t lj_ffrecord_select_mode(jit_State *J, TRef tr, TValue *tv)
 231 {
 232   if (tref_isstr(tr) && *strVdata(tv) == '#') {  /* select('#', ...) */
 233     if (strV(tv)->len == 1) {
 234       emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, strV(tv)));
 235     } else {
 236       TRef trptr = emitir(IRT(IR_STRREF, IRT_P32), tr, lj_ir_kint(J, 0));
 237       TRef trchar = emitir(IRT(IR_XLOAD, IRT_U8), trptr, IRXLOAD_READONLY);
 238       emitir(IRTG(IR_EQ, IRT_INT), trchar, lj_ir_kint(J, '#'));
 239     }
 240     return 0;
 241   } else {  /* select(n, ...) */
 242     int32_t start = argv2int(J, tv);
 243     if (start == 0) lj_trace_err(J, LJ_TRERR_BADTYPE);  /* A bit misleading. */
 244     return start;
 245   }
 246 }
 247
 248 static void LJ_FASTCALL recff_select(jit_State *J, RecordFFData *rd)
 249 {
 250   TRef tr = J->base[0];
 251   if (tr) {
 252     ptrdiff_t start = lj_ffrecord_select_mode(J, tr, &rd->argv[0]);
 253     if (start == 0) {  /* select('#', ...) */
 254       J->base[0] = lj_ir_kint(J, J->maxslot - 1);
 255     } else if (tref_isk(tr)) {  /* select(k, ...) */
 256       ptrdiff_t n = (ptrdiff_t)J->maxslot;
 257       if (start < 0) start += n;
 258       else if (start > n) start = n;
 259       rd->nres = n - start;
 260       if (start >= 1) {
 261         ptrdiff_t i;
 262         for (i = 0; i < n - start; i++)
 263           J->base[i] = J->base[start+i];
 264       }  /* else: Interpreter will throw. */
 265     } else {
 266       recff_nyiu(J);
 267     }
 268   }  /* else: Interpreter will throw. */
 269 }
 270
 271 static void LJ_FASTCALL recff_tonumber(jit_State *J, RecordFFData *rd)
 272 {
 273   TRef tr = J->base[0];
 274   TRef base = J->base[1];
 275   if (tr && base) {
 276     base = lj_opt_narrow_toint(J, base);
 277     if (!tref_isk(base) || IR(tref_ref(base))->i != 10)
 278       recff_nyiu(J);
 279   }
 280   if (tref_isnumber_str(tr)) {
 281     if (tref_isstr(tr)) {
 282       TValue tmp;
 283       if (!lj_strscan_num(strV(&rd->argv[0]), &tmp))
 284         recff_nyiu(J);  /* Would need an inverted STRTO for this case. */
 285       tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);
 286     }
 287 #if LJ_HASFFI
 288   } else if (tref_iscdata(tr)) {
 289     lj_crecord_tonumber(J, rd);
 290     return;
 291 #endif
 292   } else {
 293     tr = TREF_NIL;
 294   }
 295   J->base[0] = tr;
 296   UNUSED(rd);
 297 }
 298
 299 static TValue *recff_metacall_cp(lua_State *L, lua_CFunction dummy, void *ud)
 300 {
 301   jit_State *J = (jit_State *)ud;
 302   lj_record_tailcall(J, 0, 1);
 303   UNUSED(L); UNUSED(dummy);
 304   return NULL;
 305 }
 306
 307 static int recff_metacall(jit_State *J, RecordFFData *rd, MMS mm)
 308 {
 309   RecordIndex ix;
 310   ix.tab = J->base[0];
 311   copyTV(J->L, &ix.tabv, &rd->argv[0]);
 312   if (lj_record_mm_lookup(J, &ix, mm)) {  /* Has metamethod? */
 313     int errcode;
 314     TValue argv0;
 315     /* Temporarily insert metamethod below object. */
 316     J->base[1] = J->base[0];
 317     J->base[0] = ix.mobj;
 318     copyTV(J->L, &argv0, &rd->argv[0]);
 319     copyTV(J->L, &rd->argv[1], &rd->argv[0]);
 320     copyTV(J->L, &rd->argv[0], &ix.mobjv);
 321     /* Need to protect lj_record_tailcall because it may throw. */
 322     errcode = lj_vm_cpcall(J->L, NULL, J, recff_metacall_cp);
 323     /* Always undo Lua stack changes to avoid confusing the interpreter. */
 324     copyTV(J->L, &rd->argv[0], &argv0);
 325     if (errcode)
 326       lj_err_throw(J->L, errcode);  /* Propagate errors. */
 327     rd->nres = -1;  /* Pending call. */
 328     return 1;  /* Tailcalled to metamethod. */
 329   }
 330   return 0;
 331 }
 332
 333 static void LJ_FASTCALL recff_tostring(jit_State *J, RecordFFData *rd)
 334 {
 335   TRef tr = J->base[0];
 336   if (tref_isstr(tr)) {
 337     /* Ignore __tostring in the string base metatable. */
 338     /* Pass on result in J->base[0]. */
 339   } else if (!recff_metacall(J, rd, MM_tostring)) {
 340     if (tref_isnumber(tr)) {
 341       J->base[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr, 0);
 342     } else if (tref_ispri(tr)) {
 343       J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[tref_type(tr)]));
 344     } else {
 345       recff_nyiu(J);
 346     }
 347   }
 348 }
 349
 350 static void LJ_FASTCALL recff_ipairs_aux(jit_State *J, RecordFFData *rd)
 351 {
 352   RecordIndex ix;
 353   ix.tab = J->base[0];
 354   if (tref_istab(ix.tab)) {
 355     if (!tvisnumber(&rd->argv[1]))  /* No support for string coercion. */
 356       lj_trace_err(J, LJ_TRERR_BADTYPE);
 357     setintV(&ix.keyv, numberVint(&rd->argv[1])+1);
 358     settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
 359     ix.val = 0; ix.idxchain = 0;
 360     ix.key = lj_opt_narrow_toint(J, J->base[1]);
 361     J->base[0] = ix.key = emitir(IRTI(IR_ADD), ix.key, lj_ir_kint(J, 1));
 362     J->base[1] = lj_record_idx(J, &ix);
 363     rd->nres = tref_isnil(J->base[1]) ? 0 : 2;
 364   }  /* else: Interpreter will throw. */
 365 }
 366
 367 static void LJ_FASTCALL recff_ipairs(jit_State *J, RecordFFData *rd)
 368 {
 369   if (!(LJ_52 && recff_metacall(J, rd, MM_ipairs))) {
 370     TRef tab = J->base[0];
 371     if (tref_istab(tab)) {
 372       J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.upvalue[0]));
 373       J->base[1] = tab;
 374       J->base[2] = lj_ir_kint(J, 0);
 375       rd->nres = 3;
 376     }  /* else: Interpreter will throw. */
 377   }
 378 }
 379
 380 static void LJ_FASTCALL recff_pcall(jit_State *J, RecordFFData *rd)
 381 {
 382   if (J->maxslot >= 1) {
 383     lj_record_call(J, 0, J->maxslot - 1);
 384     rd->nres = -1;  /* Pending call. */
 385   }  /* else: Interpreter will throw. */
 386 }
 387
 388 static TValue *recff_xpcall_cp(lua_State *L, lua_CFunction dummy, void *ud)
 389 {
 390   jit_State *J = (jit_State *)ud;
 391   lj_record_call(J, 1, J->maxslot - 2);
 392   UNUSED(L); UNUSED(dummy);
 393   return NULL;
 394 }
 395
 396 static void LJ_FASTCALL recff_xpcall(jit_State *J, RecordFFData *rd)
 397 {
 398   if (J->maxslot >= 2) {
 399     TValue argv0, argv1;
 400     TRef tmp;
 401     int errcode;
 402     /* Swap function and traceback. */
 403     tmp = J->base[0]; J->base[0] = J->base[1]; J->base[1] = tmp;
 404     copyTV(J->L, &argv0, &rd->argv[0]);
 405     copyTV(J->L, &argv1, &rd->argv[1]);
 406     copyTV(J->L, &rd->argv[0], &argv1);
 407     copyTV(J->L, &rd->argv[1], &argv0);
 408     /* Need to protect lj_record_call because it may throw. */
 409     errcode = lj_vm_cpcall(J->L, NULL, J, recff_xpcall_cp);
 410     /* Always undo Lua stack swap to avoid confusing the interpreter. */
 411     copyTV(J->L, &rd->argv[0], &argv0);
 412     copyTV(J->L, &rd->argv[1], &argv1);
 413     if (errcode)
 414       lj_err_throw(J->L, errcode);  /* Propagate errors. */
 415     rd->nres = -1;  /* Pending call. */
 416   }  /* else: Interpreter will throw. */
 417 }
 418
 419 /* -- Math library fast functions ----------------------------------------- */
 420
 421 static void LJ_FASTCALL recff_math_abs(jit_State *J, RecordFFData *rd)
 422 {
 423   TRef tr = lj_ir_tonum(J, J->base[0]);
 424   J->base[0] = emitir(IRTN(IR_ABS), tr, lj_ir_knum_abs(J));
 425   UNUSED(rd);
 426 }
 427
 428 /* Record rounding functions math.floor and math.ceil. */
 429 static void LJ_FASTCALL recff_math_round(jit_State *J, RecordFFData *rd)
 430 {
 431   TRef tr = J->base[0];
 432   if (!tref_isinteger(tr)) {  /* Pass through integers unmodified. */
 433     tr = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, tr), rd->data);
 434     /* Result is integral (or NaN/Inf), but may not fit an int32_t. */
 435     if (LJ_DUALNUM) {  /* Try to narrow using a guarded conversion to int. */
 436       lua_Number n = lj_vm_foldfpm(numberVnum(&rd->argv[0]), rd->data);
 437       if (n == (lua_Number)lj_num2int(n))
 438         tr = emitir(IRTGI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_CHECK);
 439     }
 440     J->base[0] = tr;
 441   }
 442 }
 443
 444 /* Record unary math.* functions, mapped to IR_FPMATH opcode. */
 445 static void LJ_FASTCALL recff_math_unary(jit_State *J, RecordFFData *rd)
 446 {
 447   J->base[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, J->base[0]), rd->data);
 448 }
 449
 450 /* Record math.log. */
 451 static void LJ_FASTCALL recff_math_log(jit_State *J, RecordFFData *rd)
 452 {
 453   TRef tr = lj_ir_tonum(J, J->base[0]);
 454   if (J->base[1]) {
 455 #ifdef LUAJIT_NO_LOG2
 456     uint32_t fpm = IRFPM_LOG;
 457 #else
 458     uint32_t fpm = IRFPM_LOG2;
 459 #endif
 460     TRef trb = lj_ir_tonum(J, J->base[1]);
 461     tr = emitir(IRTN(IR_FPMATH), tr, fpm);
 462     trb = emitir(IRTN(IR_FPMATH), trb, fpm);
 463     trb = emitir(IRTN(IR_DIV), lj_ir_knum_one(J), trb);
 464     tr = emitir(IRTN(IR_MUL), tr, trb);
 465   } else {
 466     tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_LOG);
 467   }
 468   J->base[0] = tr;
 469   UNUSED(rd);
 470 }
 471
 472 /* Record math.atan2. */
 473 static void LJ_FASTCALL recff_math_atan2(jit_State *J, RecordFFData *rd)
 474 {
 475   TRef tr = lj_ir_tonum(J, J->base[0]);
 476   TRef tr2 = lj_ir_tonum(J, J->base[1]);
 477   J->base[0] = emitir(IRTN(IR_ATAN2), tr, tr2);
 478   UNUSED(rd);
 479 }
 480
 481 /* Record math.ldexp. */
 482 static void LJ_FASTCALL recff_math_ldexp(jit_State *J, RecordFFData *rd)
 483 {
 484   TRef tr = lj_ir_tonum(J, J->base[0]);
 485 #if LJ_TARGET_X86ORX64
 486   TRef tr2 = lj_ir_tonum(J, J->base[1]);
 487 #else
 488   TRef tr2 = lj_opt_narrow_toint(J, J->base[1]);
 489 #endif
 490   J->base[0] = emitir(IRTN(IR_LDEXP), tr, tr2);
 491   UNUSED(rd);
 492 }
 493
 494 /* Record math.asin, math.acos, math.atan. */
 495 static void LJ_FASTCALL recff_math_atrig(jit_State *J, RecordFFData *rd)
 496 {
 497   TRef y = lj_ir_tonum(J, J->base[0]);
 498   TRef x = lj_ir_knum_one(J);
 499   uint32_t ffid = rd->data;
 500   if (ffid != FF_math_atan) {
 501     TRef tmp = emitir(IRTN(IR_MUL), y, y);
 502     tmp = emitir(IRTN(IR_SUB), x, tmp);
 503     tmp = emitir(IRTN(IR_FPMATH), tmp, IRFPM_SQRT);
 504     if (ffid == FF_math_asin) { x = tmp; } else { x = y; y = tmp; }
 505   }
 506   J->base[0] = emitir(IRTN(IR_ATAN2), y, x);
 507 }
 508
 509 static void LJ_FASTCALL recff_math_htrig(jit_State *J, RecordFFData *rd)
 510 {
 511   TRef tr = lj_ir_tonum(J, J->base[0]);
 512   J->base[0] = emitir(IRTN(IR_CALLN), tr, rd->data);
 513 }
 514
 515 static void LJ_FASTCALL recff_math_modf(jit_State *J, RecordFFData *rd)
 516 {
 517   TRef tr = J->base[0];
 518   if (tref_isinteger(tr)) {
 519     J->base[0] = tr;
 520     J->base[1] = lj_ir_kint(J, 0);
 521   } else {
 522     TRef trt;
 523     tr = lj_ir_tonum(J, tr);
 524     trt = emitir(IRTN(IR_FPMATH), tr, IRFPM_TRUNC);
 525     J->base[0] = trt;
 526     J->base[1] = emitir(IRTN(IR_SUB), tr, trt);
 527   }
 528   rd->nres = 2;
 529 }
 530
 531 static void LJ_FASTCALL recff_math_degrad(jit_State *J, RecordFFData *rd)
 532 {
 533   TRef tr = lj_ir_tonum(J, J->base[0]);
 534   TRef trm = lj_ir_knum(J, numV(&J->fn->c.upvalue[0]));
 535   J->base[0] = emitir(IRTN(IR_MUL), tr, trm);
 536   UNUSED(rd);
 537 }
 538
 539 static void LJ_FASTCALL recff_math_pow(jit_State *J, RecordFFData *rd)
 540 {
 541   TRef tr = lj_ir_tonum(J, J->base[0]);
 542   if (!tref_isnumber_str(J->base[1]))
 543     lj_trace_err(J, LJ_TRERR_BADTYPE);
 544   J->base[0] = lj_opt_narrow_pow(J, tr, J->base[1], &rd->argv[1]);
 545   UNUSED(rd);
 546 }
 547
 548 static void LJ_FASTCALL recff_math_minmax(jit_State *J, RecordFFData *rd)
 549 {
 550   TRef tr = lj_ir_tonumber(J, J->base[0]);
 551   uint32_t op = rd->data;
 552   BCReg i;
 553   for (i = 1; J->base[i] != 0; i++) {
 554     TRef tr2 = lj_ir_tonumber(J, J->base[i]);
 555     IRType t = IRT_INT;
 556     if (!(tref_isinteger(tr) && tref_isinteger(tr2))) {
 557       if (tref_isinteger(tr)) tr = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
 558       if (tref_isinteger(tr2)) tr2 = emitir(IRTN(IR_CONV), tr2, IRCONV_NUM_INT);
 559       t = IRT_NUM;
 560     }
 561     tr = emitir(IRT(op, t), tr, tr2);
 562   }
 563   J->base[0] = tr;
 564 }
 565
 566 static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd)
 567 {
 568   GCudata *ud = udataV(&J->fn->c.upvalue[0]);
 569   TRef tr, one;
 570   lj_ir_kgc(J, obj2gco(ud), IRT_UDATA);  /* Prevent collection. */
 571   tr = lj_ir_call(J, IRCALL_lj_math_random_step, lj_ir_kptr(J, uddata(ud)));
 572   one = lj_ir_knum_one(J);
 573   tr = emitir(IRTN(IR_SUB), tr, one);
 574   if (J->base[0]) {
 575     TRef tr1 = lj_ir_tonum(J, J->base[0]);
 576     if (J->base[1]) {  /* d = floor(d*(r2-r1+1.0)) + r1 */
 577       TRef tr2 = lj_ir_tonum(J, J->base[1]);
 578       tr2 = emitir(IRTN(IR_SUB), tr2, tr1);
 579       tr2 = emitir(IRTN(IR_ADD), tr2, one);
 580       tr = emitir(IRTN(IR_MUL), tr, tr2);
 581       tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_FLOOR);
 582       tr = emitir(IRTN(IR_ADD), tr, tr1);
 583     } else {  /* d = floor(d*r1) + 1.0 */
 584       tr = emitir(IRTN(IR_MUL), tr, tr1);
 585       tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_FLOOR);
 586       tr = emitir(IRTN(IR_ADD), tr, one);
 587     }
 588   }
 589   J->base[0] = tr;
 590   UNUSED(rd);
 591 }
 592
 593 /* -- Bit library fast functions ------------------------------------------ */
 594
 595 /* Record unary bit.tobit, bit.bnot, bit.bswap. */
 596 static void LJ_FASTCALL recff_bit_unary(jit_State *J, RecordFFData *rd)
 597 {
 598   TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
 599   J->base[0] = (rd->data == IR_TOBIT) ? tr : emitir(IRTI(rd->data), tr, 0);
 600 }
 601
 602 /* Record N-ary bit.band, bit.bor, bit.bxor. */
 603 static void LJ_FASTCALL recff_bit_nary(jit_State *J, RecordFFData *rd)
 604 {
 605   TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
 606   uint32_t op = rd->data;
 607   BCReg i;
 608   for (i = 1; J->base[i] != 0; i++)
 609     tr = emitir(IRTI(op), tr, lj_opt_narrow_tobit(J, J->base[i]));
 610   J->base[0] = tr;
 611 }
 612
 613 /* Record bit shifts. */
 614 static void LJ_FASTCALL recff_bit_shift(jit_State *J, RecordFFData *rd)
 615 {
 616   TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
 617   TRef tsh = lj_opt_narrow_tobit(J, J->base[1]);
 618   IROp op = (IROp)rd->data;
 619   if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
 620       !tref_isk(tsh))
 621     tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31));
 622 #ifdef LJ_TARGET_UNIFYROT
 623   if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
 624     op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
 625     tsh = emitir(IRTI(IR_NEG), tsh, tsh);
 626   }
 627 #endif
 628   J->base[0] = emitir(IRTI(op), tr, tsh);
 629 }
 630
 631 /* -- String library fast functions --------------------------------------- */
 632
 633 static void LJ_FASTCALL recff_string_len(jit_State *J, RecordFFData *rd)
 634 {
 635   J->base[0] = emitir(IRTI(IR_FLOAD), lj_ir_tostr(J, J->base[0]), IRFL_STR_LEN);
 636   UNUSED(rd);
 637 }
 638
 639 /* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */
 640 static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
 641 {
 642   TRef trstr = lj_ir_tostr(J, J->base[0]);
 643   TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN);
 644   TRef tr0 = lj_ir_kint(J, 0);
 645   TRef trstart, trend;
 646   GCstr *str = argv2str(J, &rd->argv[0]);
 647   int32_t start, end;
 648   if (rd->data) {  /* string.sub(str, start [,end]) */
 649     start = argv2int(J, &rd->argv[1]);
 650     trstart = lj_opt_narrow_toint(J, J->base[1]);
 651     trend = J->base[2];
 652     if (tref_isnil(trend)) {
 653       trend = lj_ir_kint(J, -1);
 654       end = -1;
 655     } else {
 656       trend = lj_opt_narrow_toint(J, trend);
 657       end = argv2int(J, &rd->argv[2]);
 658     }
 659   } else {  /* string.byte(str, [,start [,end]]) */
 660     if (J->base[1]) {
 661       start = argv2int(J, &rd->argv[1]);
 662       trstart = lj_opt_narrow_toint(J, J->base[1]);
 663       trend = J->base[2];
 664       if (tref_isnil(trend)) {
 665         trend = trstart;
 666         end = start;
 667       } else {
 668         trend = lj_opt_narrow_toint(J, trend);
 669         end = argv2int(J, &rd->argv[2]);
 670       }
 671     } else {
 672       trend = trstart = lj_ir_kint(J, 1);
 673       end = start = 1;
 674     }
 675   }
 676   if (end < 0) {
 677     emitir(IRTGI(IR_LT), trend, tr0);
 678     trend = emitir(IRTI(IR_ADD), emitir(IRTI(IR_ADD), trlen, trend),
 679                    lj_ir_kint(J, 1));
 680     end = end+(int32_t)str->len+1;
 681   } else if ((MSize)end <= str->len) {
 682     emitir(IRTGI(IR_ULE), trend, trlen);
 683   } else {
 684     emitir(IRTGI(IR_GT), trend, trlen);
 685     end = (int32_t)str->len;
 686     trend = trlen;
 687   }
 688   if (start < 0) {
 689     emitir(IRTGI(IR_LT), trstart, tr0);
 690     trstart = emitir(IRTI(IR_ADD), trlen, trstart);
 691     start = start+(int32_t)str->len;
 692     emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), trstart, tr0);
 693     if (start < 0) {
 694       trstart = tr0;
 695       start = 0;
 696     }
 697   } else {
 698     if (start == 0) {
 699       emitir(IRTGI(IR_EQ), trstart, tr0);
 700       trstart = tr0;
 701     } else {
 702       trstart = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, -1));
 703       emitir(IRTGI(IR_GE), trstart, tr0);
 704       start--;
 705     }
 706   }
 707   if (rd->data) {  /* Return string.sub result. */
 708     if (end - start >= 0) {
 709       /* Also handle empty range here, to avoid extra traces. */
 710       TRef trptr, trslen = emitir(IRTI(IR_SUB), trend, trstart);
 711       emitir(IRTGI(IR_GE), trslen, tr0);
 712       trptr = emitir(IRT(IR_STRREF, IRT_P32), trstr, trstart);
 713       J->base[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen);
 714     } else {  /* Range underflow: return empty string. */
 715       emitir(IRTGI(IR_LT), trend, trstart);
 716       J->base[0] = lj_ir_kstr(J, lj_str_new(J->L, strdata(str), 0));
 717     }
 718   } else {  /* Return string.byte result(s). */
 719     ptrdiff_t i, len = end - start;
 720     if (len > 0) {
 721       TRef trslen = emitir(IRTI(IR_SUB), trend, trstart);
 722       emitir(IRTGI(IR_EQ), trslen, lj_ir_kint(J, (int32_t)len));
 723       if (J->baseslot + len > LJ_MAX_JSLOTS)
 724         lj_trace_err_info(J, LJ_TRERR_STACKOV);
 725       rd->nres = len;
 726       for (i = 0; i < len; i++) {
 727         TRef tmp = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, (int32_t)i));
 728         tmp = emitir(IRT(IR_STRREF, IRT_P32), trstr, tmp);
 729         J->base[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, IRXLOAD_READONLY);
 730       }
 731     } else {  /* Empty range or range underflow: return no results. */
 732       emitir(IRTGI(IR_LE), trend, trstart);
 733       rd->nres = 0;
 734     }
 735   }
 736 }
 737
 738 /* -- Table library fast functions ---------------------------------------- */
 739
 740 static void LJ_FASTCALL recff_table_getn(jit_State *J, RecordFFData *rd)
 741 {
 742   if (tref_istab(J->base[0]))
 743     J->base[0] = lj_ir_call(J, IRCALL_lj_tab_len, J->base[0]);
 744   /* else: Interpreter will throw. */
 745   UNUSED(rd);
 746 }
 747
 748 static void LJ_FASTCALL recff_table_remove(jit_State *J, RecordFFData *rd)
 749 {
 750   TRef tab = J->base[0];
 751   rd->nres = 0;
 752   if (tref_istab(tab)) {
 753     if (!J->base[1] || tref_isnil(J->base[1])) {  /* Simple pop: t[#t] = nil */
 754       TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, tab);
 755       GCtab *t = tabV(&rd->argv[0]);
 756       MSize len = lj_tab_len(t);
 757       emitir(IRTGI(len ? IR_NE : IR_EQ), trlen, lj_ir_kint(J, 0));
 758       if (len) {
 759         RecordIndex ix;
 760         ix.tab = tab;
 761         ix.key = trlen;
 762         settabV(J->L, &ix.tabv, t);
 763         setintV(&ix.keyv, len);
 764         ix.idxchain = 0;
 765         if (results_wanted(J) != 0) {  /* Specialize load only if needed. */
 766           ix.val = 0;
 767           J->base[0] = lj_record_idx(J, &ix);  /* Load previous value. */
 768           rd->nres = 1;
 769           /* Assumes ix.key/ix.tab is not modified for raw lj_record_idx(). */
 770         }
 771         ix.val = TREF_NIL;
 772         lj_record_idx(J, &ix);  /* Remove value. */
 773       }
 774     } else {  /* Complex case: remove in the middle. */
 775       recff_nyiu(J);
 776     }
 777   }  /* else: Interpreter will throw. */
 778 }
 779
 780 static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd)
 781 {
 782   RecordIndex ix;
 783   ix.tab = J->base[0];
 784   ix.val = J->base[1];
 785   rd->nres = 0;
 786   if (tref_istab(ix.tab) && ix.val) {
 787     if (!J->base[2]) {  /* Simple push: t[#t+1] = v */
 788       TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, ix.tab);
 789       GCtab *t = tabV(&rd->argv[0]);
 790       ix.key = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
 791       settabV(J->L, &ix.tabv, t);
 792       setintV(&ix.keyv, lj_tab_len(t) + 1);
 793       ix.idxchain = 0;
 794       lj_record_idx(J, &ix);  /* Set new value. */
 795     } else {  /* Complex case: insert in the middle. */
 796       recff_nyiu(J);
 797     }
 798   }  /* else: Interpreter will throw. */
 799 }
 800
 801 /* -- I/O library fast functions ------------------------------------------ */
 802
 803 /* Get FILE* for I/O function. Any I/O error aborts recording, so there's
 804 ** no need to encode the alternate cases for any of the guards.
 805 */
 806 static TRef recff_io_fp(jit_State *J, TRef *udp, int32_t id)
 807 {
 808   TRef tr, ud, fp;
 809   if (id) {  /* io.func() */
 810     tr = lj_ir_kptr(J, &J2G(J)->gcroot[id]);
 811     ud = emitir(IRT(IR_XLOAD, IRT_UDATA), tr, 0);
 812   } else {  /* fp:method() */
 813     ud = J->base[0];
 814     if (!tref_isudata(ud))
 815       lj_trace_err(J, LJ_TRERR_BADTYPE);
 816     tr = emitir(IRT(IR_FLOAD, IRT_U8), ud, IRFL_UDATA_UDTYPE);
 817     emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
 818   }
 819   *udp = ud;
 820   fp = emitir(IRT(IR_FLOAD, IRT_PTR), ud, IRFL_UDATA_FILE);
 821   emitir(IRTG(IR_NE, IRT_PTR), fp, lj_ir_knull(J, IRT_PTR));
 822   return fp;
 823 }
 824
 825 static void LJ_FASTCALL recff_io_write(jit_State *J, RecordFFData *rd)
 826 {
 827   TRef ud, fp = recff_io_fp(J, &ud, rd->data);
 828   TRef zero = lj_ir_kint(J, 0);
 829   TRef one = lj_ir_kint(J, 1);
 830   ptrdiff_t i = rd->data == 0 ? 1 : 0;
 831   for (; J->base[i]; i++) {
 832     TRef str = lj_ir_tostr(J, J->base[i]);
 833     TRef buf = emitir(IRT(IR_STRREF, IRT_P32), str, zero);
 834     TRef len = emitir(IRTI(IR_FLOAD), str, IRFL_STR_LEN);
 835     if (tref_isk(len) && IR(tref_ref(len))->i == 1) {
 836       TRef tr = emitir(IRT(IR_XLOAD, IRT_U8), buf, IRXLOAD_READONLY);
 837       tr = lj_ir_call(J, IRCALL_fputc, tr, fp);
 838       if (results_wanted(J) != 0)  /* Check result only if not ignored. */
 839         emitir(IRTGI(IR_NE), tr, lj_ir_kint(J, -1));
 840     } else {
 841       TRef tr = lj_ir_call(J, IRCALL_fwrite, buf, one, len, fp);
 842       if (results_wanted(J) != 0)  /* Check result only if not ignored. */
 843         emitir(IRTGI(IR_EQ), tr, len);
 844     }
 845   }
 846   J->base[0] = LJ_52 ? ud : TREF_TRUE;
 847 }
 848
 849 static void LJ_FASTCALL recff_io_flush(jit_State *J, RecordFFData *rd)
 850 {
 851   TRef ud, fp = recff_io_fp(J, &ud, rd->data);
 852   TRef tr = lj_ir_call(J, IRCALL_fflush, fp);
 853   if (results_wanted(J) != 0)  /* Check result only if not ignored. */
 854     emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0));
 855   J->base[0] = TREF_TRUE;
 856 }
 857
 858 /* -- Record calls to fast functions -------------------------------------- */
 859
 860 #include "lj_recdef.h"
 861
 862 static uint32_t recdef_lookup(GCfunc *fn)
 863 {
 864   if (fn->c.ffid < sizeof(recff_idmap)/sizeof(recff_idmap[0]))
 865     return recff_idmap[fn->c.ffid];
 866   else
 867     return 0;
 868 }
 869
 870 /* Record entry to a fast function or C function. */
 871 void lj_ffrecord_func(jit_State *J)
 872 {
 873   RecordFFData rd;
 874   uint32_t m = recdef_lookup(J->fn);
 875   rd.data = m & 0xff;
 876   rd.nres = 1;  /* Default is one result. */
 877   rd.argv = J->L->base;
 878   J->base[J->maxslot] = 0;  /* Mark end of arguments. */
 879   (recff_func[m >> 8])(J, &rd);  /* Call recff_* handler. */
 880   if (rd.nres >= 0) {
 881     if (J->postproc == LJ_POST_NONE) J->postproc = LJ_POST_FFRETRY;
 882     lj_record_ret(J, 0, rd.nres);
 883   }
 884 }
 885
 886 #undef IR
 887 #undef emitir
 888
 889 #endif