src/lj_obj.h

   1 /*
   2 ** LuaJIT VM tags, values and objects.
   3 ** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
   4 **
   5 ** Portions taken verbatim or adapted from the Lua interpreter.
   6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
   7 */
   8
   9 #ifndef _LJ_OBJ_H
  10 #define _LJ_OBJ_H
  11
  12 #include "lua.h"
  13 #include "lj_def.h"
  14 #include "lj_arch.h"
  15
  16 /* -- Memory references (32 bit address space) ---------------------------- */
  17
  18 /* Memory size. */
  19 typedef uint32_t MSize;
  20
  21 /* Memory reference */
  22 typedef struct MRef {
  23   uint32_t ptr32;       /* Pseudo 32 bit pointer. */
  24 } MRef;
  25
  26 #define mref(r, t)      ((t *)(void *)(uintptr_t)(r).ptr32)
  27
  28 #define setmref(r, p)   ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
  29 #define setmrefr(r, v)  ((r).ptr32 = (v).ptr32)
  30
  31 /* -- GC object references (32 bit address space) ------------------------- */
  32
  33 /* GCobj reference */
  34 typedef struct GCRef {
  35   uint32_t gcptr32;     /* Pseudo 32 bit pointer. */
  36 } GCRef;
  37
  38 /* Common GC header for all collectable objects. */
  39 #define GCHeader        GCRef nextgc; uint8_t marked; uint8_t gct
  40 /* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
  41
  42 #define gcref(r)        ((GCobj *)(uintptr_t)(r).gcptr32)
  43 #define gcrefp(r, t)    ((t *)(void *)(uintptr_t)(r).gcptr32)
  44 #define gcrefu(r)       ((r).gcptr32)
  45 #define gcrefi(r)       ((int32_t)(r).gcptr32)
  46 #define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32)
  47 #define gcnext(gc)      (gcref((gc)->gch.nextgc))
  48
  49 #define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
  50 #define setgcrefi(r, i) ((r).gcptr32 = (uint32_t)(i))
  51 #define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p))
  52 #define setgcrefnull(r) ((r).gcptr32 = 0)
  53 #define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32)
  54
  55 /* IMPORTANT NOTE:
  56 **
  57 ** All uses of the setgcref* macros MUST be accompanied with a write barrier.
  58 **
  59 ** This is to ensure the integrity of the incremental GC. The invariant
  60 ** to preserve is that a black object never points to a white object.
  61 ** I.e. never store a white object into a field of a black object.
  62 **
  63 ** It's ok to LEAVE OUT the write barrier ONLY in the following cases:
  64 ** - The source is not a GC object (NULL).
  65 ** - The target is a GC root. I.e. everything in global_State.
  66 ** - The target is a lua_State field (threads are never black).
  67 ** - The target is a stack slot, see setgcV et al.
  68 ** - The target is an open upvalue, i.e. pointing to a stack slot.
  69 ** - The target is a newly created object (i.e. marked white). But make
  70 **   sure nothing invokes the GC inbetween.
  71 ** - The target and the source are the same object (self-reference).
  72 ** - The target already contains the object (e.g. moving elements around).
  73 **
  74 ** The most common case is a store to a stack slot. All other cases where
  75 ** a barrier has been omitted are annotated with a NOBARRIER comment.
  76 **
  77 ** The same logic applies for stores to table slots (array part or hash
  78 ** part). ALL uses of lj_tab_set* require a barrier for the stored value
  79 ** *and* the stored key, based on the above rules. In practice this means
  80 ** a barrier is needed if *either* of the key or value are a GC object.
  81 **
  82 ** It's ok to LEAVE OUT the write barrier in the following special cases:
  83 ** - The stored value is nil. The key doesn't matter because it's either
  84 **   not resurrected or lj_tab_newkey() will take care of the key barrier.
  85 ** - The key doesn't matter if the *previously* stored value is guaranteed
  86 **   to be non-nil (because the key is kept alive in the table).
  87 ** - The key doesn't matter if it's guaranteed not to be part of the table,
  88 **   since lj_tab_newkey() takes care of the key barrier. This applies
  89 **   trivially to new tables, but watch out for resurrected keys. Storing
  90 **   a nil value leaves the key in the table!
  91 **
  92 ** In case of doubt use lj_gc_anybarriert() as it's rather cheap. It's used
  93 ** by the interpreter for all table stores.
  94 **
  95 ** Note: In contrast to Lua's GC, LuaJIT's GC does *not* specially mark
  96 ** dead keys in tables. The reference is left in, but it's guaranteed to
  97 ** be never dereferenced as long as the value is nil. It's ok if the key is
  98 ** freed or if any object subsequently gets the same address.
  99 **
 100 ** Not destroying dead keys helps to keep key hash slots stable. This avoids
 101 ** specialization back-off for HREFK when a value flips between nil and
 102 ** non-nil and the GC gets in the way. It also allows safely hoisting
 103 ** HREF/HREFK across GC steps. Dead keys are only removed if a table is
 104 ** resized (i.e. by NEWREF) and xREF must not be CSEd across a resize.
 105 **
 106 ** The trade-off is that a write barrier for tables must take the key into
 107 ** account, too. Implicitly resurrecting the key by storing a non-nil value
 108 ** may invalidate the incremental GC invariant.
 109 */
 110
 111 /* -- Common type definitions --------------------------------------------- */
 112
 113 /* Types for handling bytecodes. Need this here, details in lj_bc.h. */
 114 typedef uint32_t BCIns;  /* Bytecode instruction. */
 115 typedef uint32_t BCPos;  /* Bytecode position. */
 116 typedef uint32_t BCReg;  /* Bytecode register. */
 117 typedef int32_t BCLine;  /* Bytecode line number. */
 118
 119 /* Internal assembler functions. Never call these directly from C. */
 120 typedef void (*ASMFunction)(void);
 121
 122 /* Resizable string buffer. Need this here, details in lj_str.h. */
 123 typedef struct SBuf {
 124   char *buf;            /* String buffer base. */
 125   MSize n;              /* String buffer length. */
 126   MSize sz;             /* String buffer size. */
 127 } SBuf;
 128
 129 /* -- Tags and values ----------------------------------------------------- */
 130
 131 /* Frame link. */
 132 typedef union {
 133   int32_t ftsz;         /* Frame type and size of previous frame. */
 134   MRef pcr;             /* Overlaps PC for Lua frames. */
 135 } FrameLink;
 136
 137 /* Tagged value. */
 138 typedef LJ_ALIGN(8) union TValue {
 139   uint64_t u64;         /* 64 bit pattern overlaps number. */
 140   lua_Number n;         /* Number object overlaps split tag/value object. */
 141   struct {
 142     LJ_ENDIAN_LOHI(
 143       union {
 144         GCRef gcr;      /* GCobj reference (if any). */
 145         int32_t i;      /* Integer value. */
 146       };
 147     , uint32_t it;      /* Internal object tag. Must overlap MSW of number. */
 148     )
 149   };
 150   struct {
 151     LJ_ENDIAN_LOHI(
 152       GCRef func;       /* Function for next frame (or dummy L). */
 153     , FrameLink tp;     /* Link to previous frame. */
 154     )
 155   } fr;
 156   struct {
 157     LJ_ENDIAN_LOHI(
 158       uint32_t lo;      /* Lower 32 bits of number. */
 159     , uint32_t hi;      /* Upper 32 bits of number. */
 160     )
 161   } u32;
 162 } TValue;
 163
 164 typedef const TValue cTValue;
 165
 166 #define tvref(r)        (mref(r, TValue))
 167
 168 /* More external and GCobj tags for internal objects. */
 169 #define LAST_TT         LUA_TTHREAD
 170 #define LUA_TPROTO      (LAST_TT+1)
 171 #define LUA_TCDATA      (LAST_TT+2)
 172
 173 /* Internal object tags.
 174 **
 175 ** Internal tags overlap the MSW of a number object (must be a double).
 176 ** Interpreted as a double these are special NaNs. The FPU only generates
 177 ** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
 178 ** for use as internal tags. Small negative numbers are used to shorten the
 179 ** encoding of type comparisons (reg/mem against sign-ext. 8 bit immediate).
 180 **
 181 **                  ---MSW---.---LSW---
 182 ** primitive types |  itype  |         |
 183 ** lightuserdata   |  itype  |  void * |  (32 bit platforms)
 184 ** lightuserdata   |ffff|    void *    |  (64 bit platforms, 47 bit pointers)
 185 ** GC objects      |  itype  |  GCRef  |
 186 ** int (LJ_DUALNUM)|  itype  |   int   |
 187 ** number           -------double------
 188 **
 189 ** ORDER LJ_T
 190 ** Primitive types nil/false/true must be first, lightuserdata next.
 191 ** GC objects are at the end, table/userdata must be lowest.
 192 ** Also check lj_ir.h for similar ordering constraints.
 193 */
 194 #define LJ_TNIL                 (~0u)
 195 #define LJ_TFALSE               (~1u)
 196 #define LJ_TTRUE                (~2u)
 197 #define LJ_TLIGHTUD             (~3u)
 198 #define LJ_TSTR                 (~4u)
 199 #define LJ_TUPVAL               (~5u)
 200 #define LJ_TTHREAD              (~6u)
 201 #define LJ_TPROTO               (~7u)
 202 #define LJ_TFUNC                (~8u)
 203 #define LJ_TTRACE               (~9u)
 204 #define LJ_TCDATA               (~10u)
 205 #define LJ_TTAB                 (~11u)
 206 #define LJ_TUDATA               (~12u)
 207 /* This is just the canonical number type used in some places. */
 208 #define LJ_TNUMX                (~13u)
 209
 210 /* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
 211 #if LJ_64
 212 #define LJ_TISNUM               0xfffeffffu
 213 #else
 214 #define LJ_TISNUM               LJ_TNUMX
 215 #endif
 216 #define LJ_TISTRUECOND          LJ_TFALSE
 217 #define LJ_TISPRI               LJ_TTRUE
 218 #define LJ_TISGCV               (LJ_TSTR+1)
 219 #define LJ_TISTABUD             LJ_TTAB
 220
 221 /* -- String object ------------------------------------------------------- */
 222
 223 /* String object header. String payload follows. */
 224 typedef struct GCstr {
 225   GCHeader;
 226   uint8_t reserved;     /* Used by lexer for fast lookup of reserved words. */
 227   uint8_t unused;
 228   MSize hash;           /* Hash of string. */
 229   MSize len;            /* Size of string. */
 230 } GCstr;
 231
 232 #define strref(r)       (&gcref((r))->str)
 233 #define strdata(s)      ((const char *)((s)+1))
 234 #define strdatawr(s)    ((char *)((s)+1))
 235 #define strVdata(o)     strdata(strV(o))
 236 #define sizestring(s)   (sizeof(struct GCstr)+(s)->len+1)
 237
 238 /* -- Userdata object ----------------------------------------------------- */
 239
 240 /* Userdata object. Payload follows. */
 241 typedef struct GCudata {
 242   GCHeader;
 243   uint8_t udtype;       /* Userdata type. */
 244   uint8_t unused2;
 245   GCRef env;            /* Should be at same offset in GCfunc. */
 246   MSize len;            /* Size of payload. */
 247   GCRef metatable;      /* Must be at same offset in GCtab. */
 248   uint32_t align1;      /* To force 8 byte alignment of the payload. */
 249 } GCudata;
 250
 251 /* Userdata types. */
 252 enum {
 253   UDTYPE_USERDATA,      /* Regular userdata. */
 254   UDTYPE_IO_FILE,       /* I/O library FILE. */
 255   UDTYPE_FFI_CLIB,      /* FFI C library namespace. */
 256   UDTYPE__MAX
 257 };
 258
 259 #define uddata(u)       ((void *)((u)+1))
 260 #define sizeudata(u)    (sizeof(struct GCudata)+(u)->len)
 261
 262 /* -- C data object ------------------------------------------------------- */
 263
 264 /* C data object. Payload follows. */
 265 typedef struct GCcdata {
 266   GCHeader;
 267   uint16_t ctypeid;     /* C type ID. */
 268 } GCcdata;
 269
 270 /* Prepended to variable-sized or realigned C data objects. */
 271 typedef struct GCcdataVar {
 272   uint16_t offset;      /* Offset to allocated memory (relative to GCcdata). */
 273   uint16_t extra;       /* Extra space allocated (incl. GCcdata + GCcdatav). */
 274   MSize len;            /* Size of payload. */
 275 } GCcdataVar;
 276
 277 #define cdataptr(cd)    ((void *)((cd)+1))
 278 #define cdataisv(cd)    ((cd)->marked & 0x80)
 279 #define cdatav(cd)      ((GCcdataVar *)((char *)(cd) - sizeof(GCcdataVar)))
 280 #define cdatavlen(cd)   check_exp(cdataisv(cd), cdatav(cd)->len)
 281 #define sizecdatav(cd)  (cdatavlen(cd) + cdatav(cd)->extra)
 282 #define memcdatav(cd)   ((void *)((char *)(cd) - cdatav(cd)->offset))
 283
 284 /* -- Prototype object ---------------------------------------------------- */
 285
 286 #define SCALE_NUM_GCO   ((int32_t)sizeof(lua_Number)/sizeof(GCRef))
 287 #define round_nkgc(n)   (((n) + SCALE_NUM_GCO-1) & ~(SCALE_NUM_GCO-1))
 288
 289 typedef struct GCproto {
 290   GCHeader;
 291   uint8_t numparams;    /* Number of parameters. */
 292   uint8_t framesize;    /* Fixed frame size. */
 293   MSize sizebc;         /* Number of bytecode instructions. */
 294   GCRef gclist;
 295   MRef k;               /* Split constant array (points to the middle). */
 296   MRef uv;              /* Upvalue list. local slot|0x8000 or parent uv idx. */
 297   MSize sizekgc;        /* Number of collectable constants. */
 298   MSize sizekn;         /* Number of lua_Number constants. */
 299   MSize sizept;         /* Total size including colocated arrays. */
 300   uint8_t sizeuv;       /* Number of upvalues. */
 301   uint8_t flags;        /* Miscellaneous flags (see below). */
 302   uint16_t trace;       /* Anchor for chain of root traces. */
 303   /* ------ The following fields are for debugging/tracebacks only ------ */
 304   GCRef chunkname;      /* Name of the chunk this function was defined in. */
 305   BCLine firstline;     /* First line of the function definition. */
 306   BCLine numline;       /* Number of lines for the function definition. */
 307   MRef lineinfo;        /* Compressed map from bytecode ins. to source line. */
 308   MRef uvinfo;          /* Upvalue names. */
 309   MRef varinfo;         /* Names and compressed extents of local variables. */
 310 } GCproto;
 311
 312 /* Flags for prototype. */
 313 #define PROTO_CHILD             0x01    /* Has child prototypes. */
 314 #define PROTO_VARARG            0x02    /* Vararg function. */
 315 #define PROTO_FFI               0x04    /* Uses BC_KCDATA for FFI datatypes. */
 316 #define PROTO_NOJIT             0x08    /* JIT disabled for this function. */
 317 #define PROTO_ILOOP             0x10    /* Patched bytecode with ILOOP etc. */
 318 /* Only used during parsing. */
 319 #define PROTO_HAS_RETURN        0x20    /* Already emitted a return. */
 320 #define PROTO_FIXUP_RETURN      0x40    /* Need to fixup emitted returns. */
 321 /* Top bits used for counting created closures. */
 322 #define PROTO_CLCOUNT           0x20    /* Base of saturating 3 bit counter. */
 323 #define PROTO_CLC_BITS          3
 324 #define PROTO_CLC_POLY          (3*PROTO_CLCOUNT)  /* Polymorphic threshold. */
 325
 326 #define PROTO_UV_LOCAL          0x8000  /* Upvalue for local slot. */
 327 #define PROTO_UV_IMMUTABLE      0x4000  /* Immutable upvalue. */
 328
 329 #define proto_kgc(pt, idx) \
 330   check_exp((uintptr_t)(intptr_t)(idx) >= (uintptr_t)-(intptr_t)(pt)->sizekgc, \
 331             gcref(mref((pt)->k, GCRef)[(idx)]))
 332 #define proto_knumtv(pt, idx) \
 333   check_exp((uintptr_t)(idx) < (pt)->sizekn, &mref((pt)->k, TValue)[(idx)])
 334 #define proto_bc(pt)            ((BCIns *)((char *)(pt) + sizeof(GCproto)))
 335 #define proto_bcpos(pt, pc)     ((BCPos)((pc) - proto_bc(pt)))
 336 #define proto_uv(pt)            (mref((pt)->uv, uint16_t))
 337
 338 #define proto_chunkname(pt)     (strref((pt)->chunkname))
 339 #define proto_chunknamestr(pt)  (strdata(proto_chunkname((pt))))
 340 #define proto_lineinfo(pt)      (mref((pt)->lineinfo, const void))
 341 #define proto_uvinfo(pt)        (mref((pt)->uvinfo, const uint8_t))
 342 #define proto_varinfo(pt)       (mref((pt)->varinfo, const uint8_t))
 343
 344 /* -- Upvalue object ------------------------------------------------------ */
 345
 346 typedef struct GCupval {
 347   GCHeader;
 348   uint8_t closed;       /* Set if closed (i.e. uv->v == &uv->u.value). */
 349   uint8_t immutable;    /* Immutable value. */
 350   union {
 351     TValue tv;          /* If closed: the value itself. */
 352     struct {            /* If open: double linked list, anchored at thread. */
 353       GCRef prev;
 354       GCRef next;
 355     };
 356   };
 357   MRef v;               /* Points to stack slot (open) or above (closed). */
 358   uint32_t dhash;       /* Disambiguation hash: dh1 != dh2 => cannot alias. */
 359 } GCupval;
 360
 361 #define uvprev(uv_)     (&gcref((uv_)->prev)->uv)
 362 #define uvnext(uv_)     (&gcref((uv_)->next)->uv)
 363 #define uvval(uv_)      (mref((uv_)->v, TValue))
 364
 365 /* -- Function object (closures) ------------------------------------------ */
 366
 367 /* Common header for functions. env should be at same offset in GCudata. */
 368 #define GCfuncHeader \
 369   GCHeader; uint8_t ffid; uint8_t nupvalues; \
 370   GCRef env; GCRef gclist; MRef pc
 371
 372 typedef struct GCfuncC {
 373   GCfuncHeader;
 374   lua_CFunction f;      /* C function to be called. */
 375   TValue upvalue[1];    /* Array of upvalues (TValue). */
 376 } GCfuncC;
 377
 378 typedef struct GCfuncL {
 379   GCfuncHeader;
 380   GCRef uvptr[1];       /* Array of _pointers_ to upvalue objects (GCupval). */
 381 } GCfuncL;
 382
 383 typedef union GCfunc {
 384   GCfuncC c;
 385   GCfuncL l;
 386 } GCfunc;
 387
 388 #define FF_LUA          0
 389 #define FF_C            1
 390 #define isluafunc(fn)   ((fn)->c.ffid == FF_LUA)
 391 #define iscfunc(fn)     ((fn)->c.ffid == FF_C)
 392 #define isffunc(fn)     ((fn)->c.ffid > FF_C)
 393 #define funcproto(fn) \
 394   check_exp(isluafunc(fn), (GCproto *)(mref((fn)->l.pc, char)-sizeof(GCproto)))
 395 #define sizeCfunc(n)    (sizeof(GCfuncC)-sizeof(TValue)+sizeof(TValue)*(n))
 396 #define sizeLfunc(n)    (sizeof(GCfuncL)-sizeof(GCRef)+sizeof(GCRef)*(n))
 397
 398 /* -- Table object -------------------------------------------------------- */
 399
 400 /* Hash node. */
 401 typedef struct Node {
 402   TValue val;           /* Value object. Must be first field. */
 403   TValue key;           /* Key object. */
 404   MRef next;            /* Hash chain. */
 405   MRef freetop;         /* Top of free elements (stored in t->node[0]). */
 406 } Node;
 407
 408 LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
 409
 410 typedef struct GCtab {
 411   GCHeader;
 412   uint8_t nomm;         /* Negative cache for fast metamethods. */
 413   int8_t colo;          /* Array colocation. */
 414   MRef array;           /* Array part. */
 415   GCRef gclist;
 416   GCRef metatable;      /* Must be at same offset in GCudata. */
 417   MRef node;            /* Hash part. */
 418   uint32_t asize;       /* Size of array part (keys [0, asize-1]). */
 419   uint32_t hmask;       /* Hash part mask (size of hash part - 1). */
 420 } GCtab;
 421
 422 #define sizetabcolo(n)  ((n)*sizeof(TValue) + sizeof(GCtab))
 423 #define tabref(r)       (&gcref((r))->tab)
 424 #define noderef(r)      (mref((r), Node))
 425 #define nextnode(n)     (mref((n)->next, Node))
 426
 427 /* -- State objects ------------------------------------------------------- */
 428
 429 /* VM states. */
 430 enum {
 431   LJ_VMST_INTERP,       /* Interpreter. */
 432   LJ_VMST_C,            /* C function. */
 433   LJ_VMST_GC,           /* Garbage collector. */
 434   LJ_VMST_EXIT,         /* Trace exit handler. */
 435   LJ_VMST_RECORD,       /* Trace recorder. */
 436   LJ_VMST_OPT,          /* Optimizer. */
 437   LJ_VMST_ASM,          /* Assembler. */
 438   LJ_VMST__MAX
 439 };
 440
 441 #define setvmstate(g, st)       ((g)->vmstate = ~LJ_VMST_##st)
 442
 443 /* Metamethods. ORDER MM */
 444 #ifdef LJ_HASFFI
 445 #define MMDEF_FFI(_) _(new)
 446 #else
 447 #define MMDEF_FFI(_)
 448 #endif
 449
 450 #if LJ_52 || LJ_HASFFI
 451 #define MMDEF_PAIRS(_) _(pairs) _(ipairs)
 452 #else
 453 #define MMDEF_PAIRS(_)
 454 #define MM_pairs        255
 455 #define MM_ipairs       255
 456 #endif
 457
 458 #define MMDEF(_) \
 459   _(index) _(newindex) _(gc) _(mode) _(eq) _(len) \
 460   /* Only the above (fast) metamethods are negative cached (max. 8). */ \
 461   _(lt) _(le) _(concat) _(call) \
 462   /* The following must be in ORDER ARITH. */ \
 463   _(add) _(sub) _(mul) _(div) _(mod) _(pow) _(unm) \
 464   /* The following are used in the standard libraries. */ \
 465   _(metatable) _(tostring) MMDEF_FFI(_) MMDEF_PAIRS(_)
 466
 467 typedef enum {
 468 #define MMENUM(name)    MM_##name,
 469 MMDEF(MMENUM)
 470 #undef MMENUM
 471   MM__MAX,
 472   MM____ = MM__MAX,
 473   MM_FAST = MM_len
 474 } MMS;
 475
 476 /* GC root IDs. */
 477 typedef enum {
 478   GCROOT_MMNAME,        /* Metamethod names. */
 479   GCROOT_MMNAME_LAST = GCROOT_MMNAME + MM__MAX-1,
 480   GCROOT_BASEMT,        /* Metatables for base types. */
 481   GCROOT_BASEMT_NUM = GCROOT_BASEMT + ~LJ_TNUMX,
 482   GCROOT_IO_INPUT,      /* Userdata for default I/O input file. */
 483   GCROOT_IO_OUTPUT,     /* Userdata for default I/O output file. */
 484   GCROOT_MAX
 485 } GCRootID;
 486
 487 #define basemt_it(g, it)        ((g)->gcroot[GCROOT_BASEMT+~(it)])
 488 #define basemt_obj(g, o)        ((g)->gcroot[GCROOT_BASEMT+itypemap(o)])
 489 #define mmname_str(g, mm)       (strref((g)->gcroot[GCROOT_MMNAME+(mm)]))
 490
 491 typedef struct GCState {
 492   MSize total;          /* Memory currently allocated. */
 493   MSize threshold;      /* Memory threshold. */
 494   uint8_t currentwhite; /* Current white color. */
 495   uint8_t state;        /* GC state. */
 496   uint8_t unused1;
 497   uint8_t unused2;
 498   MSize sweepstr;       /* Sweep position in string table. */
 499   GCRef root;           /* List of all collectable objects. */
 500   MRef sweep;           /* Sweep position in root list. */
 501   GCRef gray;           /* List of gray objects. */
 502   GCRef grayagain;      /* List of objects for atomic traversal. */
 503   GCRef weak;           /* List of weak tables (to be cleared). */
 504   GCRef mmudata;        /* List of userdata (to be finalized). */
 505   MSize stepmul;        /* Incremental GC step granularity. */
 506   MSize debt;           /* Debt (how much GC is behind schedule). */
 507   MSize estimate;       /* Estimate of memory actually in use. */
 508   MSize pause;          /* Pause between successive GC cycles. */
 509 } GCState;
 510
 511 /* Global state, shared by all threads of a Lua universe. */
 512 typedef struct global_State {
 513   GCRef *strhash;       /* String hash table (hash chain anchors). */
 514   MSize strmask;        /* String hash mask (size of hash table - 1). */
 515   MSize strnum;         /* Number of strings in hash table. */
 516   lua_Alloc allocf;     /* Memory allocator. */
 517   void *allocd;         /* Memory allocator data. */
 518   GCState gc;           /* Garbage collector. */
 519   SBuf tmpbuf;          /* Temporary buffer for string concatenation. */
 520   Node nilnode;         /* Fallback 1-element hash part (nil key and value). */
 521   GCstr strempty;       /* Empty string. */
 522   uint8_t stremptyz;    /* Zero terminator of empty string. */
 523   uint8_t hookmask;     /* Hook mask. */
 524   uint8_t dispatchmode; /* Dispatch mode. */
 525   uint8_t vmevmask;     /* VM event mask. */
 526   GCRef mainthref;      /* Link to main thread. */
 527   TValue registrytv;    /* Anchor for registry. */
 528   TValue tmptv, tmptv2; /* Temporary TValues. */
 529   GCupval uvhead;       /* Head of double-linked list of all open upvalues. */
 530   int32_t hookcount;    /* Instruction hook countdown. */
 531   int32_t hookcstart;   /* Start count for instruction hook counter. */
 532   lua_Hook hookf;       /* Hook function. */
 533   lua_CFunction wrapf;  /* Wrapper for C function calls. */
 534   lua_CFunction panic;  /* Called as a last resort for errors. */
 535   volatile int32_t vmstate;  /* VM state or current JIT code trace number. */
 536   BCIns bc_cfunc_int;   /* Bytecode for internal C function calls. */
 537   BCIns bc_cfunc_ext;   /* Bytecode for external C function calls. */
 538   GCRef jit_L;          /* Current JIT code lua_State or NULL. */
 539   MRef jit_base;        /* Current JIT code L->base. */
 540   MRef ctype_state;     /* Pointer to C type state. */
 541   GCRef gcroot[GCROOT_MAX];  /* GC roots. */
 542 } global_State;
 543
 544 #define mainthread(g)   (&gcref(g->mainthref)->th)
 545 #define niltv(L) \
 546   check_exp(tvisnil(&G(L)->nilnode.val), &G(L)->nilnode.val)
 547 #define niltvg(g) \
 548   check_exp(tvisnil(&(g)->nilnode.val), &(g)->nilnode.val)
 549
 550 /* Hook management. Hook event masks are defined in lua.h. */
 551 #define HOOK_EVENTMASK          0x0f
 552 #define HOOK_ACTIVE             0x10
 553 #define HOOK_ACTIVE_SHIFT       4
 554 #define HOOK_VMEVENT            0x20
 555 #define HOOK_GC                 0x40
 556 #define hook_active(g)          ((g)->hookmask & HOOK_ACTIVE)
 557 #define hook_enter(g)           ((g)->hookmask |= HOOK_ACTIVE)
 558 #define hook_entergc(g)         ((g)->hookmask |= (HOOK_ACTIVE|HOOK_GC))
 559 #define hook_vmevent(g)         ((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT))
 560 #define hook_leave(g)           ((g)->hookmask &= ~HOOK_ACTIVE)
 561 #define hook_save(g)            ((g)->hookmask & ~HOOK_EVENTMASK)
 562 #define hook_restore(g, h) \
 563   ((g)->hookmask = ((g)->hookmask & HOOK_EVENTMASK) | (h))
 564
 565 /* Per-thread state object. */
 566 struct lua_State {
 567   GCHeader;
 568   uint8_t dummy_ffid;   /* Fake FF_C for curr_funcisL() on dummy frames. */
 569   uint8_t status;       /* Thread status. */
 570   MRef glref;           /* Link to global state. */
 571   GCRef gclist;         /* GC chain. */
 572   TValue *base;         /* Base of currently executing function. */
 573   TValue *top;          /* First free slot in the stack. */
 574   MRef maxstack;        /* Last free slot in the stack. */
 575   MRef stack;           /* Stack base. */
 576   GCRef openupval;      /* List of open upvalues in the stack. */
 577   GCRef env;            /* Thread environment (table of globals). */
 578   void *cframe;         /* End of C stack frame chain. */
 579   MSize stacksize;      /* True stack size (incl. LJ_STACK_EXTRA). */
 580 };
 581
 582 #define G(L)                    (mref(L->glref, global_State))
 583 #define registry(L)             (&G(L)->registrytv)
 584
 585 /* Macros to access the currently executing (Lua) function. */
 586 #define curr_func(L)            (&gcref((L->base-1)->fr.func)->fn)
 587 #define curr_funcisL(L)         (isluafunc(curr_func(L)))
 588 #define curr_proto(L)           (funcproto(curr_func(L)))
 589 #define curr_topL(L)            (L->base + curr_proto(L)->framesize)
 590 #define curr_top(L)             (curr_funcisL(L) ? curr_topL(L) : L->top)
 591
 592 /* -- GC object definition and conversions -------------------------------- */
 593
 594 /* GC header for generic access to common fields of GC objects. */
 595 typedef struct GChead {
 596   GCHeader;
 597   uint8_t unused1;
 598   uint8_t unused2;
 599   GCRef env;
 600   GCRef gclist;
 601   GCRef metatable;
 602 } GChead;
 603
 604 /* The env field SHOULD be at the same offset for all GC objects. */
 605 LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCfuncL, env));
 606 LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCudata, env));
 607
 608 /* The metatable field MUST be at the same offset for all GC objects. */
 609 LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCtab, metatable));
 610 LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCudata, metatable));
 611
 612 /* The gclist field MUST be at the same offset for all GC objects. */
 613 LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(lua_State, gclist));
 614 LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist));
 615 LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCfuncL, gclist));
 616 LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtab, gclist));
 617
 618 typedef union GCobj {
 619   GChead gch;
 620   GCstr str;
 621   GCupval uv;
 622   lua_State th;
 623   GCproto pt;
 624   GCfunc fn;
 625   GCcdata cd;
 626   GCtab tab;
 627   GCudata ud;
 628 } GCobj;
 629
 630 /* Macros to convert a GCobj pointer into a specific value. */
 631 #define gco2str(o)      check_exp((o)->gch.gct == ~LJ_TSTR, &(o)->str)
 632 #define gco2uv(o)       check_exp((o)->gch.gct == ~LJ_TUPVAL, &(o)->uv)
 633 #define gco2th(o)       check_exp((o)->gch.gct == ~LJ_TTHREAD, &(o)->th)
 634 #define gco2pt(o)       check_exp((o)->gch.gct == ~LJ_TPROTO, &(o)->pt)
 635 #define gco2func(o)     check_exp((o)->gch.gct == ~LJ_TFUNC, &(o)->fn)
 636 #define gco2cd(o)       check_exp((o)->gch.gct == ~LJ_TCDATA, &(o)->cd)
 637 #define gco2tab(o)      check_exp((o)->gch.gct == ~LJ_TTAB, &(o)->tab)
 638 #define gco2ud(o)       check_exp((o)->gch.gct == ~LJ_TUDATA, &(o)->ud)
 639
 640 /* Macro to convert any collectable object into a GCobj pointer. */
 641 #define obj2gco(v)      ((GCobj *)(v))
 642
 643 /* -- TValue getters/setters ---------------------------------------------- */
 644
 645 #ifdef LUA_USE_ASSERT
 646 #include "lj_gc.h"
 647 #endif
 648
 649 /* Macros to test types. */
 650 #define itype(o)        ((o)->it)
 651 #define tvisnil(o)      (itype(o) == LJ_TNIL)
 652 #define tvisfalse(o)    (itype(o) == LJ_TFALSE)
 653 #define tvistrue(o)     (itype(o) == LJ_TTRUE)
 654 #define tvisbool(o)     (tvisfalse(o) || tvistrue(o))
 655 #if LJ_64
 656 #define tvislightud(o)  (((int32_t)itype(o) >> 15) == -2)
 657 #else
 658 #define tvislightud(o)  (itype(o) == LJ_TLIGHTUD)
 659 #endif
 660 #define tvisstr(o)      (itype(o) == LJ_TSTR)
 661 #define tvisfunc(o)     (itype(o) == LJ_TFUNC)
 662 #define tvisthread(o)   (itype(o) == LJ_TTHREAD)
 663 #define tvisproto(o)    (itype(o) == LJ_TPROTO)
 664 #define tviscdata(o)    (itype(o) == LJ_TCDATA)
 665 #define tvistab(o)      (itype(o) == LJ_TTAB)
 666 #define tvisudata(o)    (itype(o) == LJ_TUDATA)
 667 #define tvisnumber(o)   (itype(o) <= LJ_TISNUM)
 668 #define tvisint(o)      (LJ_DUALNUM && itype(o) == LJ_TISNUM)
 669 #define tvisnum(o)      (itype(o) < LJ_TISNUM)
 670
 671 #define tvistruecond(o) (itype(o) < LJ_TISTRUECOND)
 672 #define tvispri(o)      (itype(o) >= LJ_TISPRI)
 673 #define tvistabud(o)    (itype(o) <= LJ_TISTABUD)  /* && !tvisnum() */
 674 #define tvisgcv(o)      ((itype(o) - LJ_TISGCV) > (LJ_TNUMX - LJ_TISGCV))
 675
 676 /* Special macros to test numbers for NaN, +0, -0, +1 and raw equality. */
 677 #define tvisnan(o)      ((o)->n != (o)->n)
 678 #if LJ_64
 679 #define tviszero(o)     (((o)->u64 << 1) == 0)
 680 #else
 681 #define tviszero(o)     (((o)->u32.lo | ((o)->u32.hi << 1)) == 0)
 682 #endif
 683 #define tvispzero(o)    ((o)->u64 == 0)
 684 #define tvismzero(o)    ((o)->u64 == U64x(80000000,00000000))
 685 #define tvispone(o)     ((o)->u64 == U64x(3ff00000,00000000))
 686 #define rawnumequal(o1, o2)     ((o1)->u64 == (o2)->u64)
 687
 688 /* Macros to convert type ids. */
 689 #if LJ_64
 690 #define itypemap(o) \
 691   (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
 692 #else
 693 #define itypemap(o)     (tvisnumber(o) ? ~LJ_TNUMX : ~itype(o))
 694 #endif
 695
 696 /* Macros to get tagged values. */
 697 #define gcval(o)        (gcref((o)->gcr))
 698 #define boolV(o)        check_exp(tvisbool(o), (LJ_TFALSE - (o)->it))
 699 #if LJ_64
 700 #define lightudV(o) \
 701   check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff)))
 702 #else
 703 #define lightudV(o)     check_exp(tvislightud(o), gcrefp((o)->gcr, void))
 704 #endif
 705 #define gcV(o)          check_exp(tvisgcv(o), gcval(o))
 706 #define strV(o)         check_exp(tvisstr(o), &gcval(o)->str)
 707 #define funcV(o)        check_exp(tvisfunc(o), &gcval(o)->fn)
 708 #define threadV(o)      check_exp(tvisthread(o), &gcval(o)->th)
 709 #define protoV(o)       check_exp(tvisproto(o), &gcval(o)->pt)
 710 #define cdataV(o)       check_exp(tviscdata(o), &gcval(o)->cd)
 711 #define tabV(o)         check_exp(tvistab(o), &gcval(o)->tab)
 712 #define udataV(o)       check_exp(tvisudata(o), &gcval(o)->ud)
 713 #define numV(o)         check_exp(tvisnum(o), (o)->n)
 714 #define intV(o)         check_exp(tvisint(o), (int32_t)(o)->i)
 715
 716 /* Macros to set tagged values. */
 717 #define setitype(o, i)          ((o)->it = (i))
 718 #define setnilV(o)              ((o)->it = LJ_TNIL)
 719 #define setboolV(o, x)          ((o)->it = LJ_TFALSE-(uint32_t)(x))
 720
 721 static LJ_AINLINE void setlightudV(TValue *o, void *p)
 722 {
 723 #if LJ_64
 724   o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
 725 #else
 726   setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
 727 #endif
 728 }
 729
 730 #if LJ_64
 731 #define checklightudptr(L, p) \
 732   (((uint64_t)(p) >> 47) ? (lj_err_msg(L, LJ_ERR_BADLU), NULL) : (p))
 733 #define setcont(o, f) \
 734   ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
 735 #else
 736 #define checklightudptr(L, p)   (p)
 737 #define setcont(o, f)           setlightudV((o), (void *)(f))
 738 #endif
 739
 740 #define tvchecklive(L, o) \
 741   UNUSED(L), lua_assert(!tvisgcv(o) || \
 742   ((~itype(o) == gcval(o)->gch.gct) && !isdead(G(L), gcval(o))))
 743
 744 static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t itype)
 745 {
 746   setgcref(o->gcr, v); setitype(o, itype); tvchecklive(L, o);
 747 }
 748
 749 #define define_setV(name, type, tag) \
 750 static LJ_AINLINE void name(lua_State *L, TValue *o, type *v) \
 751 { \
 752   setgcV(L, o, obj2gco(v), tag); \
 753 }
 754 define_setV(setstrV, GCstr, LJ_TSTR)
 755 define_setV(setthreadV, lua_State, LJ_TTHREAD)
 756 define_setV(setprotoV, GCproto, LJ_TPROTO)
 757 define_setV(setfuncV, GCfunc, LJ_TFUNC)
 758 define_setV(setcdataV, GCcdata, LJ_TCDATA)
 759 define_setV(settabV, GCtab, LJ_TTAB)
 760 define_setV(setudataV, GCudata, LJ_TUDATA)
 761
 762 #define setnumV(o, x)           ((o)->n = (x))
 763 #define setnanV(o)              ((o)->u64 = U64x(fff80000,00000000))
 764 #define setpinfV(o)             ((o)->u64 = U64x(7ff00000,00000000))
 765 #define setminfV(o)             ((o)->u64 = U64x(fff00000,00000000))
 766
 767 static LJ_AINLINE void setintV(TValue *o, int32_t i)
 768 {
 769 #if LJ_DUALNUM
 770   o->i = (uint32_t)i; setitype(o, LJ_TISNUM);
 771 #else
 772   o->n = (lua_Number)i;
 773 #endif
 774 }
 775
 776 static LJ_AINLINE void setint64V(TValue *o, int64_t i)
 777 {
 778   if (LJ_DUALNUM && LJ_LIKELY(i == (int64_t)(int32_t)i))
 779     setintV(o, (int32_t)i);
 780   else
 781     setnumV(o, (lua_Number)i);
 782 }
 783
 784 #if LJ_64
 785 #define setintptrV(o, i)        setint64V((o), (i))
 786 #else
 787 #define setintptrV(o, i)        setintV((o), (i))
 788 #endif
 789
 790 /* Copy tagged values. */
 791 static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2)
 792 {
 793   *o1 = *o2; tvchecklive(L, o1);
 794 }
 795
 796 /* -- Number to integer conversion ---------------------------------------- */
 797
 798 #if LJ_SOFTFP
 799 LJ_ASMF int32_t lj_vm_tobit(double x);
 800 #endif
 801
 802 static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
 803 {
 804 #if LJ_SOFTFP
 805   return lj_vm_tobit(n);
 806 #else
 807   TValue o;
 808   o.n = n + 6755399441055744.0;  /* 2^52 + 2^51 */
 809   return (int32_t)o.u32.lo;
 810 #endif
 811 }
 812
 813 #if LJ_TARGET_X86 && !defined(__SSE2__)
 814 #define lj_num2int(n)   lj_num2bit((n))
 815 #else
 816 #define lj_num2int(n)   ((int32_t)(n))
 817 #endif
 818
 819 static LJ_AINLINE uint64_t lj_num2u64(lua_Number n)
 820 {
 821 #ifdef _MSC_VER
 822   if (n >= 9223372036854775808.0)  /* They think it's a feature. */
 823     return (uint64_t)(int64_t)(n - 18446744073709551616.0);
 824   else
 825 #endif
 826     return (uint64_t)n;
 827 }
 828
 829 static LJ_AINLINE int32_t numberVint(cTValue *o)
 830 {
 831   if (LJ_LIKELY(tvisint(o)))
 832     return intV(o);
 833   else
 834     return lj_num2int(numV(o));
 835 }
 836
 837 static LJ_AINLINE lua_Number numberVnum(cTValue *o)
 838 {
 839   if (LJ_UNLIKELY(tvisint(o)))
 840     return (lua_Number)intV(o);
 841   else
 842     return numV(o);
 843 }
 844
 845 /* -- Miscellaneous object handling --------------------------------------- */
 846
 847 /* Names and maps for internal and external object tags. */
 848 LJ_DATA const char *const lj_obj_typename[1+LUA_TCDATA+1];
 849 LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
 850
 851 #define lj_typename(o)  (lj_obj_itypename[itypemap(o)])
 852
 853 /* Compare two objects without calling metamethods. */
 854 LJ_FUNC int lj_obj_equal(cTValue *o1, cTValue *o2);
 855
 856 #endif