6 #define TMP_ADDR 0x0001
10 #define LOC_SYM 0x0800
11 #define LOC_LOCAL 0x1000
12 #define LOC_MASK 0xff00
51 #define MIN(a, b) ((a) < (b) ? (a) : (b))
53 #define TMP_BT(t) ((t)->flags & TMP_ADDR ? 8 : (t)->bt)
54 #define TMP_REG(t) ((t)->flags & LOC_REG ? (t)->addr : reg_get(~0))
55 #define TMP_REG2(t, r) ((t)->flags & LOC_REG && (t)->addr != r ? \
56 (t)->addr : reg_get(~(1 << r)))
57 #define BT_TMPBT(bt) (BT_SZ(bt) >= 4 ? (bt) : (bt) & BT_SIGNED | 4)
59 static char buf
[SECSIZE
];
63 static long spsub_addr
;
66 #define TMP(i) (&tmps[ntmp - 1 - (i)])
70 long off
; /* used for LOC_SYM; offset from a symbol address */
78 static struct tmp
*regs
[NREGS
];
79 static int tmpregs
[] = {R_RAX
, R_RDI
, R_RSI
, R_RDX
, R_RCX
, R_R8
, R_R9
};
81 #define MAXRET (1 << 8)
83 static long ret
[MAXRET
];
89 static void putint(char *s
, long n
, int l
)
97 static void os(char *s
, int n
)
105 static void oi(long n
, int l
)
115 static long codeaddr(void)
120 #define OP2(o2, o1) (0x0100 | ((o2) << 8) | (o1))
121 #define O2(op) (((op) >> 8) & 0xff)
122 #define O1(op) ((op) & 0xff)
124 static void o_op(int op
, int r1
, int r2
, unsigned bt
)
134 if (sz
== 1 && (r1
== R_RSI
|| r1
== R_RDI
||
135 r2
== R_RSI
|| r2
== R_RDI
))
137 /* hack: for movxx ops, bt does not represent the second arg */
138 if (op
& 0x100 && O2(op
) == 0x0f && (O1(op
) & 0xf7) == 0xb6 &&
139 (r2
== R_RSI
|| r2
== R_RDI
))
147 oi(sz
== 1 ? O1(op
) & ~0x1 : O1(op
), 1);
150 static void memop(int op
, int src
, int base
, int off
, unsigned bt
)
152 int dis
= off
== (char) off
? 1 : 4;
153 int mod
= dis
== 4 ? 2 : 1;
154 o_op(op
, src
, base
, bt
);
157 oi((mod
<< 6) | ((src
& 0x07) << 3) | (base
& 0x07), 1);
162 static void regop(int op
, int src
, int dst
, unsigned bt
)
164 o_op(op
, src
, dst
, bt
);
165 oi((3 << 6) | (src
<< 3) | (dst
& 0x07), 1);
168 static long sp_push(int size
)
176 #define LOC_NEW(f, l) (((f) & ~LOC_MASK) | (l))
178 static void tmp_mem(struct tmp
*tmp
)
181 if (!(tmp
->flags
& LOC_REG
))
185 tmp
->addr
= -sp_push(8);
186 memop(MOV_R2X
, src
, R_RBP
, tmp
->addr
, BT_TMPBT(TMP_BT(tmp
)));
188 tmp
->flags
= LOC_NEW(tmp
->flags
, LOC_MEM
);
191 static int movxx_x2r(int bt
)
195 o2
= bt
& BT_SIGNED
? 0xbe : 0xb6;
197 o2
= bt
& BT_SIGNED
? 0xbf : 0xb7;
198 return OP2(0x0f, o2
);
203 static void mov_r2r(int r1
, int r2
, unsigned bt1
, unsigned bt2
)
205 int s1
= bt1
& BT_SIGNED
;
206 int s2
= bt2
& BT_SIGNED
;
207 int sz1
= BT_SZ(bt1
);
208 int sz2
= BT_SZ(bt2
);
209 if (sz2
< 4 && (sz1
> sz2
|| s1
!= s2
)) {
210 regop(movxx_x2r(bt2
), r1
, r2
, 4);
213 if (sz1
== 4 && sz2
== 8 && s1
) {
214 regop(MOVSXD
, r2
, r1
, sz2
);
217 if (r1
!= r2
|| sz1
> sz2
)
218 regop(MOV_R2X
, r1
, r2
, BT_TMPBT(bt2
));
221 static void mov_m2r(int dst
, int base
, int off
, int bt1
, int bt2
)
223 if (BT_SZ(bt1
) < 4) {
224 memop(movxx_x2r(bt1
), dst
, base
, off
,
225 bt1
& BT_SIGNED
&& BT_SZ(bt2
) == 8 ? 8 : 4);
226 mov_r2r(dst
, dst
, bt1
, bt2
);
228 memop(MOV_M2R
, dst
, base
, off
, bt1
);
229 mov_r2r(dst
, dst
, bt1
, bt2
);
233 static void num_cast(struct tmp
*t
, unsigned bt
)
235 if (!(bt
& BT_SIGNED
) && BT_SZ(bt
) != 8)
236 t
->addr
&= ((1l << (long) (BT_SZ(bt
) * 8)) - 1);
237 if (bt
& BT_SIGNED
&& BT_SZ(bt
) != 8 &&
238 t
->addr
> (1l << (BT_SZ(bt
) * 8 - 1)))
239 t
->addr
= -((1l << (BT_SZ(bt
) * 8)) - t
->addr
);
243 static void num_reg(int reg
, unsigned bt
, long num
)
245 int op
= MOV_I2R
+ (reg
& 7);
246 if (BT_SZ(bt
) == 8 && num
>= 0 && num
== (unsigned) num
)
248 o_op(op
, 0, reg
, bt
);
252 static void tmp_reg(struct tmp
*tmp
, int dst
, unsigned bt
, int deref
)
254 if (deref
&& tmp
->flags
& TMP_ADDR
)
255 tmp
->flags
&= ~TMP_ADDR
;
258 if (tmp
->flags
& LOC_NUM
) {
260 tmp
->bt
= BT_TMPBT(bt
);
261 num_reg(dst
, tmp
->bt
, tmp
->addr
);
264 tmp
->flags
= LOC_NEW(tmp
->flags
, LOC_REG
);
266 if (tmp
->flags
& LOC_SYM
) {
267 regop(MOV_I2X
, 0, dst
, 4);
269 out_rela(tmp
->addr
, codeaddr(), 0);
273 tmp
->flags
= LOC_NEW(tmp
->flags
, LOC_REG
);
275 if (tmp
->flags
& LOC_REG
) {
277 mov_m2r(dst
, tmp
->addr
, 0, tmp
->bt
, bt
);
279 mov_r2r(tmp
->addr
, dst
, TMP_BT(tmp
),
280 tmp
->flags
& TMP_ADDR
? 8 : bt
);
281 regs
[tmp
->addr
] = NULL
;
283 if (tmp
->flags
& LOC_LOCAL
) {
285 mov_m2r(dst
, R_RBP
, tmp
->addr
, tmp
->bt
, bt
);
287 memop(LEA_M2R
, dst
, R_RBP
, tmp
->addr
, 8);
289 if (tmp
->flags
& LOC_MEM
) {
290 int nbt
= deref
? 8 : TMP_BT(tmp
);
291 mov_m2r(dst
, R_RBP
, tmp
->addr
, nbt
, nbt
);
293 mov_m2r(dst
, dst
, 0, tmp
->bt
, bt
);
296 tmp
->bt
= tmp
->flags
& TMP_ADDR
? bt
: BT_TMPBT(bt
);
298 tmp
->flags
= LOC_NEW(tmp
->flags
, LOC_REG
);
301 static void reg_free(int reg
)
306 for (i
= 0; i
< ARRAY_SIZE(tmpregs
); i
++)
307 if (!regs
[tmpregs
[i
]]) {
308 tmp_reg(regs
[reg
], tmpregs
[i
], regs
[reg
]->bt
, 0);
314 static void reg_for(int reg
, struct tmp
*t
)
316 if (regs
[reg
] && regs
[reg
] != t
)
320 static void tmp_mv(struct tmp
*t
, int reg
)
323 tmp_reg(t
, reg
, t
->bt
, 0);
326 static void tmp_to(struct tmp
*t
, int reg
, int bt
)
329 tmp_reg(t
, reg
, bt
? bt
: t
->bt
, 1);
332 static void tmp_drop(int n
)
335 for (i
= ntmp
- n
; i
< ntmp
; i
++)
336 if (tmps
[i
].flags
& LOC_REG
)
337 regs
[tmps
[i
].addr
] = NULL
;
342 static int tmp_pop(int reg
, int bt
)
344 struct tmp
*t
= TMP(0);
350 static struct tmp
*tmp_new(void)
353 return &tmps
[ntmp
++];
356 static void tmp_push(int reg
, unsigned bt
)
358 struct tmp
*t
= tmp_new();
365 void o_local(long addr
, unsigned bt
)
367 struct tmp
*t
= tmp_new();
370 t
->flags
= LOC_LOCAL
| TMP_ADDR
;
373 void o_num(long num
, unsigned bt
)
375 struct tmp
*t
= tmp_new();
381 void o_symaddr(long addr
, unsigned bt
)
383 struct tmp
*t
= tmp_new();
386 t
->flags
= LOC_SYM
| TMP_ADDR
;
390 void o_tmpdrop(int n
)
392 if (n
== -1 || n
> ntmp
)
402 #define FORK_REG R_RAX
404 /* make sure tmps remain intact after a conditional expression */
408 for (i
= 0; i
< ntmp
- 1; i
++)
412 void o_forkpush(void)
417 void o_forkjoin(void)
419 tmp_push(FORK_REG
, 0);
424 struct tmp
*t1
= TMP(0);
425 struct tmp
*t2
= TMP(1);
427 memcpy(&t
, t1
, sizeof(t
));
428 memcpy(t1
, t2
, sizeof(t
));
429 memcpy(t2
, &t
, sizeof(t
));
430 if (t1
->flags
& LOC_REG
)
432 if (t2
->flags
& LOC_REG
)
436 static int reg_get(int mask
)
439 for (i
= 0; i
< ARRAY_SIZE(tmpregs
); i
++)
440 if ((1 << tmpregs
[i
]) & mask
&& !regs
[tmpregs
[i
]])
442 for (i
= 0; i
< ARRAY_SIZE(tmpregs
); i
++)
443 if ((1 << tmpregs
[i
]) & mask
) {
444 reg_free(tmpregs
[i
]);
450 void tmp_copy(struct tmp
*t1
)
452 struct tmp
*t2
= tmp_new();
453 memcpy(t2
, t1
, sizeof(*t1
));
454 if (!(t1
->flags
& (LOC_REG
| LOC_MEM
)))
456 if (t1
->flags
& LOC_MEM
) {
457 tmp_reg(t2
, reg_get(~0), t2
->bt
, 0);
458 } else if (t1
->flags
& LOC_REG
) {
459 t2
->addr
= reg_get(~(1 << t1
->addr
));
460 regop(MOV_R2X
, t1
->addr
, t2
->addr
, BT_TMPBT(TMP_BT(t1
)));
470 void o_cast(unsigned bt
)
472 struct tmp
*t
= TMP(0);
476 if (t
->flags
& LOC_NUM
) {
484 long o_func_beg(char *name
, int global
)
486 long addr
= out_func_beg(name
, global
);
488 os("\x55", 1); /* push %rbp */
489 os("\x48\x89\xe5", 3); /* mov %rsp, %rbp */
496 memset(regs
, 0, sizeof(regs
));
497 os("\x48\x81\xec", 3); /* sub $xxx, %rsp */
498 spsub_addr
= codeaddr();
503 void o_deref(unsigned bt
)
505 struct tmp
*t
= TMP(0);
506 if (t
->flags
& TMP_ADDR
)
507 tmp_to(t
, TMP_REG(t
), t
->bt
);
509 t
->flags
|= TMP_ADDR
;
514 struct tmp
*t
= TMP(0);
515 tmp_to(t
, TMP_REG(t
), t
->bt
);
518 static unsigned bt_op(unsigned bt1
, unsigned bt2
)
520 unsigned s1
= BT_SZ(bt1
);
521 unsigned s2
= BT_SZ(bt2
);
522 unsigned bt
= (bt1
| bt2
) & BT_SIGNED
| (s1
> s2
? s1
: s2
);
526 #define TMP_CONST(t) ((t)->flags & LOC_NUM && !((t)->flags & TMP_ADDR))
527 #define LOCAL_PTR(t) ((t)->flags & LOC_LOCAL && !((t)->flags & TMP_ADDR))
528 #define SYM_PTR(t) ((t)->flags & LOC_SYM && !((t)->flags & TMP_ADDR))
530 int o_popnum(long *c
)
532 struct tmp
*t
= TMP(0);
540 static int c_op(long (*cop
)(long a
, unsigned bt
), unsigned bt
)
542 struct tmp
*t1
= TMP(0);
548 ret
= cop(t1
->addr
, bt
);
554 static void shx(int uop
, int sop
)
556 struct tmp
*t2
= TMP(1);
557 int r2
= TMP_REG2(t2
, R_RCX
);
561 regop(SHX_REG
, bt
& BT_SIGNED
? sop
: uop
, r2
, BT_TMPBT(bt
));
565 static int mulop(int uop
, int sop
, int reg
)
567 struct tmp
*t1
= TMP(0);
568 struct tmp
*t2
= TMP(1);
569 int bt
= bt_op(t1
->bt
, t2
->bt
);
570 if (t1
->flags
& LOC_REG
&& t1
->addr
!= R_RAX
&& t1
->addr
!= R_RDX
)
573 tmp_to(t2
, R_RAX
, bt
);
577 o_op(0x99, R_RAX
, R_RDX
, bt
);
579 regop(XOR_R2X
, R_RDX
, R_RDX
, bt
);
582 regop(MUL_A2X
, bt
& BT_SIGNED
? sop
: uop
, reg
, BT_TMPBT(t2
->bt
));
588 tmps
[ntmp
- 1].flags
&= ~TMP_ADDR
;
589 tmps
[ntmp
- 1].bt
= 8;
592 void o_ret(unsigned bt
)
597 os("\x31\xc0", 2); /* xor %eax, %eax */
598 ret
[nret
++] = o_jmp(0);
601 static void inc(int op
)
603 struct tmp
*t
= TMP(0);
606 if (t
->flags
& LOC_LOCAL
) {
614 memop(INC_X
, op
, reg
, off
, t
->bt
);
619 if (cmp_last
== codeaddr()) {
620 buf
[cmp_setl
+ 1] ^= 0x01;
622 o_num(0, 4 | BT_SIGNED
);
629 struct tmp
*t
= TMP(0);
631 unsigned bt
= BT_TMPBT(t
->bt
);
634 regop(NOT_REG
, id
, reg
, bt
);
637 void o_func_end(void)
640 for (i
= 0; i
< nret
; i
++)
642 os("\xc9\xc3", 2); /* leave; ret; */
643 putint(buf
+ spsub_addr
, (maxsp
+ 7) & ~0x07, 4);
644 out_func_end(buf
, cur
- buf
);
647 long o_mklocal(int size
)
649 return sp_push((size
+ 7) & ~0x07);
652 void o_rmlocal(long addr
, int sz
)
657 static int arg_regs
[] = {R_RDI
, R_RSI
, R_RDX
, R_RCX
, R_R8
, R_R9
};
659 #define R_NARGS ARRAY_SIZE(arg_regs)
661 long o_arg(int i
, unsigned bt
)
665 addr
= o_mklocal(BT_SZ(bt
));
666 memop(MOV_R2X
, arg_regs
[i
], R_RBP
, -addr
, BT_TMPBT(bt
));
668 addr
= -8 * (i
- R_NARGS
+ 2);
673 void o_assign(unsigned bt
)
675 struct tmp
*t1
= TMP(0);
676 struct tmp
*t2
= TMP(1);
677 int r1
= TMP_REG(t1
);
680 tmp_to(t1
, r1
, BT_TMPBT(bt
));
681 if (t2
->flags
& LOC_LOCAL
) {
685 reg
= TMP_REG2(t2
, r1
);
690 memop(MOV_R2X
, r1
, reg
, off
, bt
);
694 static long cu(int op
, long i
)
706 static int c_uop(int op
)
708 struct tmp
*t1
= TMP(0);
712 o_num(cu(op
, t1
->addr
), t1
->bt
);
716 static long cb(int op
, long a
, long b
, int *bt
)
741 return (unsigned long) a
>> b
;
763 static int c_bop(int op
)
765 struct tmp
*t1
= TMP(0);
766 struct tmp
*t2
= TMP(1);
767 int locals
= LOCAL_PTR(t1
) + LOCAL_PTR(t2
);
768 int syms
= SYM_PTR(t1
) + SYM_PTR(t2
);
769 int nums
= TMP_CONST(t1
) + TMP_CONST(t2
);
771 if (syms
== 2 || syms
&& locals
|| syms
+ nums
+ locals
!= 2)
774 bt
= syms
? 8 : bt_op(t1
->bt
, t2
->bt
);
780 long o1
= SYM_PTR(t1
) ? t1
->off
: t1
->addr
;
781 long o2
= SYM_PTR(t2
) ? t2
->off
: t2
->addr
;
782 long ret
= cb(op
, o2
, o1
, &bt
);
788 long ret
= cb(op
, t2
->addr
, t1
->addr
, &bt
);
807 o_neg(op
== O_NEG
? 3 : 2);
819 static int binop(int op
, int *reg
)
821 struct tmp
*t1
= TMP(0);
822 struct tmp
*t2
= TMP(1);
824 unsigned bt
= bt_op(t1
->bt
, t2
->bt
);
826 r2
= TMP_REG2(t2
, r1
);
829 regop(op
, r2
, r1
, bt
);
834 static void bin_add(int op
)
836 /* opcode for O_ADD, O_SUB, O_AND, O_OR, O_XOR */
837 static int rx
[] = {0x03, 0x2b, 0x23, 0x0b, 0x33};
839 int bt
= binop(rx
[op
& 0x0f], ®
);
843 static void bin_shx(int op
)
845 if ((op
& 0xff) == O_SHL
)
851 static void bin_mul(int op
)
853 if ((op
& 0xff) == O_MUL
)
854 tmp_push(R_RAX
, mulop(4, 5, R_RDX
));
855 if ((op
& 0xff) == O_DIV
)
856 tmp_push(R_RAX
, mulop(6, 7, R_RCX
));
857 if ((op
& 0xff) == O_MOD
)
858 tmp_push(R_RDX
, mulop(6, 7, R_RCX
));
861 static void o_cmp(int uop
, int sop
)
863 struct tmp
*t1
= TMP(0);
864 struct tmp
*t2
= TMP(1);
865 char set
[] = "\x0f\x00\xc0";
868 if (regs
[R_RAX
] && regs
[R_RAX
] != t1
&& regs
[R_RAX
] != t2
)
870 bt
= binop(CMP_R2X
, ®
);
871 set
[1] = bt
& BT_SIGNED
? sop
: uop
;
872 cmp_setl
= codeaddr();
873 os(set
, 3); /* setl %al */
874 os("\x0f\xb6\xc0", 3); /* movzbl %al, %eax */
875 tmp_push(R_RAX
, 4 | BT_SIGNED
);
876 cmp_last
= codeaddr();
879 static void bin_cmp(int op
)
903 static void o_bopset(int op
)
908 o_assign(TMP(1)->bt
);
913 if (!(op
& O_SET
) && !c_bop(op
))
919 if ((op
& 0xf0) == 0x00)
921 if ((op
& 0xf0) == 0x10)
923 if ((op
& 0xf0) == 0x20)
925 if ((op
& 0xf0) == 0x30)
929 void o_memcpy(int sz
)
931 struct tmp
*t0
= TMP(-1);
932 struct tmp
*t1
= TMP(0);
933 struct tmp
*t2
= TMP(1);
935 tmp_to(t0
, R_RCX
, 0);
938 os("\xf3\xa4", 2); /* rep movs */
942 void o_memset(int x
, int sz
)
944 struct tmp
*t0
= TMP(-2);
945 struct tmp
*t1
= TMP(-1);
946 struct tmp
*t2
= TMP(0);
949 tmp_to(t0
, R_RAX
, 0);
950 tmp_to(t1
, R_RCX
, 0);
952 os("\xf3\xaa", 2); /* rep stosb */
961 static long jx(int x
, long addr
)
965 os(op
, 2); /* jx $addr */
966 oi(addr
- codeaddr() - 4, 4);
967 return codeaddr() - 4;
970 static long jxtest(int x
, long addr
)
972 int bt
= tmp_pop(R_RAX
, 0);
973 regop(TEST_R2R
, R_RAX
, R_RAX
, bt
);
977 static long jxcmp(long addr
, int inv
)
980 if (codeaddr() != cmp_last
)
983 cur
= buf
+ cmp_setl
;
984 x
= (unsigned char) buf
[cmp_setl
+ 1];
985 return jx((inv
? x
: x
^ 0x01) & ~0x10, addr
);
990 long ret
= jxcmp(addr
, 0);
991 return ret
!= -1 ? ret
: jxtest(0x84, addr
);
994 long o_jnz(long addr
)
996 long ret
= jxcmp(addr
, 1);
997 return ret
!= -1 ? ret
: jxtest(0x85, addr
);
1000 long o_jmp(long addr
)
1002 os("\xe9", 1); /* jmp $addr */
1003 oi(addr
- codeaddr() - 4, 4);
1004 return codeaddr() - 4;
1007 void o_filljmp2(long addr
, long jmpdst
)
1009 putint(buf
+ addr
, jmpdst
- addr
- 4, 4);
1012 void o_filljmp(long addr
)
1014 o_filljmp2(addr
, codeaddr());
1017 void o_call(int argc
, unsigned *bt
, unsigned ret_bt
)
1021 for (i
= 0; i
< ARRAY_SIZE(tmpregs
); i
++)
1022 if (regs
[tmpregs
[i
]] && regs
[tmpregs
[i
]] - tmps
< ntmp
- argc
)
1023 tmp_mem(regs
[tmpregs
[i
]]);
1024 if (argc
> R_NARGS
) {
1025 long addr
= sp_push(8 * (argc
- R_NARGS
));
1026 for (i
= argc
- 1; i
>= R_NARGS
; --i
) {
1027 int reg
= TMP_REG(TMP(0));
1028 tmp_pop(reg
, bt
[i
]);
1029 memop(MOV_R2X
, reg
, R_RBP
,
1030 -(addr
- (i
- R_NARGS
) * 8), BT_TMPBT(bt
[i
]));
1033 for (i
= MIN(argc
, R_NARGS
) - 1; i
>= 0; i
--)
1034 tmp_pop(arg_regs
[i
], BT_TMPBT(bt
[i
]));
1036 if (t
->flags
& LOC_SYM
) {
1037 os("\x31\xc0", 2); /* xor %eax, %eax */
1038 os("\xe8", 1); /* call $x */
1040 out_rela(t
->addr
, codeaddr(), 1);
1044 tmp_mv(TMP(0), R_RAX
);
1046 regop(CALL_REG
, 2, R_RAX
, 4);
1049 tmp_push(R_RAX
, ret_bt
);
1062 void o_datset(long addr
, int off
, unsigned bt
)
1064 struct tmp
*t
= TMP(0);
1065 if (t
->flags
& LOC_NUM
&& !(t
->flags
& TMP_ADDR
)) {
1067 out_datcpy(addr
, off
, (void *) &t
->addr
, BT_SZ(bt
));
1069 if (t
->flags
& LOC_SYM
&& !(t
->flags
& TMP_ADDR
)) {
1070 out_datrela(t
->addr
, addr
, off
);
1071 out_datcpy(addr
, off
, (void *) &t
->off
, BT_SZ(bt
));