?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
nfp: bpf: allow stack accesses via modified stack registers
[linux-2.6/btrfs-unstable.git] / drivers / net / ethernet / netronome / nfp / bpf / jit.c
blob073e382cba047e96cf1efe2430b1a2b5b87c7bc2
1 /*
2 * Copyright (C) 2016 Netronome Systems, Inc.
3 *
4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree or the BSD 2-Clause License provided below. You have the
7 * option to license this software under the complete terms of either license.
8 *
9 * The BSD 2-Clause License:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * 1. Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * 2. Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #define pr_fmt(fmt) "NFP net bpf: " fmt
35
36 #include <linux/kernel.h>
37 #include <linux/bpf.h>
38 #include <linux/filter.h>
39 #include <linux/pkt_cls.h>
40 #include <linux/unistd.h>
41
42 #include "main.h"
43 #include "../nfp_asm.h"
44
45 /* --- NFP prog --- */
46 /* Foreach "multiple" entries macros provide pos and next<n> pointers.
47 * It's safe to modify the next pointers (but not pos).
48 */
49 #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \
50 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
51 next = list_next_entry(pos, l); \
52 &(nfp_prog)->insns != &pos->l && \
53 &(nfp_prog)->insns != &next->l; \
54 pos = nfp_meta_next(pos), \
55 next = nfp_meta_next(pos))
56
57 #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \
58 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
59 next = list_next_entry(pos, l), \
60 next2 = list_next_entry(next, l); \
61 &(nfp_prog)->insns != &pos->l && \
62 &(nfp_prog)->insns != &next->l && \
63 &(nfp_prog)->insns != &next2->l; \
64 pos = nfp_meta_next(pos), \
65 next = nfp_meta_next(pos), \
66 next2 = nfp_meta_next(next))
67
68 static bool
69 nfp_meta_has_next(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
70 {
71 return meta->l.next != &nfp_prog->insns;
72 }
73
74 static bool
75 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
76 {
77 return meta->l.prev != &nfp_prog->insns;
78 }
79
80 static void nfp_prog_free(struct nfp_prog *nfp_prog)
81 {
82 struct nfp_insn_meta *meta, *tmp;
83
84 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
85 list_del(&meta->l);
86 kfree(meta);
87 }
88 kfree(nfp_prog);
89 }
90
91 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
92 {
93 if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) {
94 nfp_prog->error = -ENOSPC;
95 return;
96 }
97
98 nfp_prog->prog[nfp_prog->prog_len] = insn;
99 nfp_prog->prog_len++;
100 }
101
102 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
103 {
104 return nfp_prog->start_off + nfp_prog->prog_len;
105 }
106
107 static unsigned int
108 nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset)
109 {
110 return offset - nfp_prog->start_off;
111 }
112
113 /* --- Emitters --- */
114 static void
115 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
116 u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync)
117 {
118 enum cmd_ctx_swap ctx;
119 u64 insn;
120
121 if (sync)
122 ctx = CMD_CTX_SWAP;
123 else
124 ctx = CMD_CTX_NO_SWAP;
125
126 insn = FIELD_PREP(OP_CMD_A_SRC, areg) |
127 FIELD_PREP(OP_CMD_CTX, ctx) |
128 FIELD_PREP(OP_CMD_B_SRC, breg) |
129 FIELD_PREP(OP_CMD_TOKEN, cmd_tgt_act[op].token) |
130 FIELD_PREP(OP_CMD_XFER, xfer) |
131 FIELD_PREP(OP_CMD_CNT, size) |
132 FIELD_PREP(OP_CMD_SIG, sync) |
133 FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) |
134 FIELD_PREP(OP_CMD_MODE, mode);
135
136 nfp_prog_push(nfp_prog, insn);
137 }
138
139 static void
140 emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
141 u8 mode, u8 xfer, swreg lreg, swreg rreg, u8 size, bool sync)
142 {
143 struct nfp_insn_re_regs reg;
144 int err;
145
146 err = swreg_to_restricted(reg_none(), lreg, rreg, &reg, false);
147 if (err) {
148 nfp_prog->error = err;
149 return;
150 }
151 if (reg.swap) {
152 pr_err("cmd can't swap arguments\n");
153 nfp_prog->error = -EFAULT;
154 return;
155 }
156 if (reg.dst_lmextn || reg.src_lmextn) {
157 pr_err("cmd can't use LMextn\n");
158 nfp_prog->error = -EFAULT;
159 return;
160 }
161
162 __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync);
163 }
164
165 static void
166 __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
167 enum br_ctx_signal_state css, u16 addr, u8 defer)
168 {
169 u16 addr_lo, addr_hi;
170 u64 insn;
171
172 addr_lo = addr & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
173 addr_hi = addr != addr_lo;
174
175 insn = OP_BR_BASE |
176 FIELD_PREP(OP_BR_MASK, mask) |
177 FIELD_PREP(OP_BR_EV_PIP, ev_pip) |
178 FIELD_PREP(OP_BR_CSS, css) |
179 FIELD_PREP(OP_BR_DEFBR, defer) |
180 FIELD_PREP(OP_BR_ADDR_LO, addr_lo) |
181 FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
182
183 nfp_prog_push(nfp_prog, insn);
184 }
185
186 static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer)
187 {
188 if (defer > 2) {
189 pr_err("BUG: branch defer out of bounds %d\n", defer);
190 nfp_prog->error = -EFAULT;
191 return;
192 }
193 __emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer);
194 }
195
196 static void
197 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
198 {
199 __emit_br(nfp_prog, mask,
200 mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
201 BR_CSS_NONE, addr, defer);
202 }
203
204 static void
205 __emit_br_byte(struct nfp_prog *nfp_prog, u8 areg, u8 breg, bool imm8,
206 u8 byte, bool equal, u16 addr, u8 defer, bool src_lmextn)
207 {
208 u16 addr_lo, addr_hi;
209 u64 insn;
210
211 addr_lo = addr & (OP_BB_ADDR_LO >> __bf_shf(OP_BB_ADDR_LO));
212 addr_hi = addr != addr_lo;
213
214 insn = OP_BBYTE_BASE |
215 FIELD_PREP(OP_BB_A_SRC, areg) |
216 FIELD_PREP(OP_BB_BYTE, byte) |
217 FIELD_PREP(OP_BB_B_SRC, breg) |
218 FIELD_PREP(OP_BB_I8, imm8) |
219 FIELD_PREP(OP_BB_EQ, equal) |
220 FIELD_PREP(OP_BB_DEFBR, defer) |
221 FIELD_PREP(OP_BB_ADDR_LO, addr_lo) |
222 FIELD_PREP(OP_BB_ADDR_HI, addr_hi) |
223 FIELD_PREP(OP_BB_SRC_LMEXTN, src_lmextn);
224
225 nfp_prog_push(nfp_prog, insn);
226 }
227
228 static void
229 emit_br_byte_neq(struct nfp_prog *nfp_prog,
230 swreg src, u8 imm, u8 byte, u16 addr, u8 defer)
231 {
232 struct nfp_insn_re_regs reg;
233 int err;
234
235 err = swreg_to_restricted(reg_none(), src, reg_imm(imm), &reg, true);
236 if (err) {
237 nfp_prog->error = err;
238 return;
239 }
240
241 __emit_br_byte(nfp_prog, reg.areg, reg.breg, reg.i8, byte, false, addr,
242 defer, reg.src_lmextn);
243 }
244
245 static void
246 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
247 enum immed_width width, bool invert,
248 enum immed_shift shift, bool wr_both,
249 bool dst_lmextn, bool src_lmextn)
250 {
251 u64 insn;
252
253 insn = OP_IMMED_BASE |
254 FIELD_PREP(OP_IMMED_A_SRC, areg) |
255 FIELD_PREP(OP_IMMED_B_SRC, breg) |
256 FIELD_PREP(OP_IMMED_IMM, imm_hi) |
257 FIELD_PREP(OP_IMMED_WIDTH, width) |
258 FIELD_PREP(OP_IMMED_INV, invert) |
259 FIELD_PREP(OP_IMMED_SHIFT, shift) |
260 FIELD_PREP(OP_IMMED_WR_AB, wr_both) |
261 FIELD_PREP(OP_IMMED_SRC_LMEXTN, src_lmextn) |
262 FIELD_PREP(OP_IMMED_DST_LMEXTN, dst_lmextn);
263
264 nfp_prog_push(nfp_prog, insn);
265 }
266
267 static void
268 emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm,
269 enum immed_width width, bool invert, enum immed_shift shift)
270 {
271 struct nfp_insn_ur_regs reg;
272 int err;
273
274 if (swreg_type(dst) == NN_REG_IMM) {
275 nfp_prog->error = -EFAULT;
276 return;
277 }
278
279 err = swreg_to_unrestricted(dst, dst, reg_imm(imm & 0xff), &reg);
280 if (err) {
281 nfp_prog->error = err;
282 return;
283 }
284
285 __emit_immed(nfp_prog, reg.areg, reg.breg, imm >> 8, width,
286 invert, shift, reg.wr_both,
287 reg.dst_lmextn, reg.src_lmextn);
288 }
289
290 static void
291 __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
292 enum shf_sc sc, u8 shift,
293 u16 areg, enum shf_op op, u16 breg, bool i8, bool sw, bool wr_both,
294 bool dst_lmextn, bool src_lmextn)
295 {
296 u64 insn;
297
298 if (!FIELD_FIT(OP_SHF_SHIFT, shift)) {
299 nfp_prog->error = -EFAULT;
300 return;
301 }
302
303 if (sc == SHF_SC_L_SHF)
304 shift = 32 - shift;
305
306 insn = OP_SHF_BASE |
307 FIELD_PREP(OP_SHF_A_SRC, areg) |
308 FIELD_PREP(OP_SHF_SC, sc) |
309 FIELD_PREP(OP_SHF_B_SRC, breg) |
310 FIELD_PREP(OP_SHF_I8, i8) |
311 FIELD_PREP(OP_SHF_SW, sw) |
312 FIELD_PREP(OP_SHF_DST, dst) |
313 FIELD_PREP(OP_SHF_SHIFT, shift) |
314 FIELD_PREP(OP_SHF_OP, op) |
315 FIELD_PREP(OP_SHF_DST_AB, dst_ab) |
316 FIELD_PREP(OP_SHF_WR_AB, wr_both) |
317 FIELD_PREP(OP_SHF_SRC_LMEXTN, src_lmextn) |
318 FIELD_PREP(OP_SHF_DST_LMEXTN, dst_lmextn);
319
320 nfp_prog_push(nfp_prog, insn);
321 }
322
323 static void
324 emit_shf(struct nfp_prog *nfp_prog, swreg dst,
325 swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc, u8 shift)
326 {
327 struct nfp_insn_re_regs reg;
328 int err;
329
330 err = swreg_to_restricted(dst, lreg, rreg, &reg, true);
331 if (err) {
332 nfp_prog->error = err;
333 return;
334 }
335
336 __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift,
337 reg.areg, op, reg.breg, reg.i8, reg.swap, reg.wr_both,
338 reg.dst_lmextn, reg.src_lmextn);
339 }
340
341 static void
342 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
343 u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both,
344 bool dst_lmextn, bool src_lmextn)
345 {
346 u64 insn;
347
348 insn = OP_ALU_BASE |
349 FIELD_PREP(OP_ALU_A_SRC, areg) |
350 FIELD_PREP(OP_ALU_B_SRC, breg) |
351 FIELD_PREP(OP_ALU_DST, dst) |
352 FIELD_PREP(OP_ALU_SW, swap) |
353 FIELD_PREP(OP_ALU_OP, op) |
354 FIELD_PREP(OP_ALU_DST_AB, dst_ab) |
355 FIELD_PREP(OP_ALU_WR_AB, wr_both) |
356 FIELD_PREP(OP_ALU_SRC_LMEXTN, src_lmextn) |
357 FIELD_PREP(OP_ALU_DST_LMEXTN, dst_lmextn);
358
359 nfp_prog_push(nfp_prog, insn);
360 }
361
362 static void
363 emit_alu(struct nfp_prog *nfp_prog, swreg dst,
364 swreg lreg, enum alu_op op, swreg rreg)
365 {
366 struct nfp_insn_ur_regs reg;
367 int err;
368
369 err = swreg_to_unrestricted(dst, lreg, rreg, &reg);
370 if (err) {
371 nfp_prog->error = err;
372 return;
373 }
374
375 __emit_alu(nfp_prog, reg.dst, reg.dst_ab,
376 reg.areg, op, reg.breg, reg.swap, reg.wr_both,
377 reg.dst_lmextn, reg.src_lmextn);
378 }
379
380 static void
381 __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc,
382 u8 areg, u8 bmask, u8 breg, u8 shift, bool imm8,
383 bool zero, bool swap, bool wr_both,
384 bool dst_lmextn, bool src_lmextn)
385 {
386 u64 insn;
387
388 insn = OP_LDF_BASE |
389 FIELD_PREP(OP_LDF_A_SRC, areg) |
390 FIELD_PREP(OP_LDF_SC, sc) |
391 FIELD_PREP(OP_LDF_B_SRC, breg) |
392 FIELD_PREP(OP_LDF_I8, imm8) |
393 FIELD_PREP(OP_LDF_SW, swap) |
394 FIELD_PREP(OP_LDF_ZF, zero) |
395 FIELD_PREP(OP_LDF_BMASK, bmask) |
396 FIELD_PREP(OP_LDF_SHF, shift) |
397 FIELD_PREP(OP_LDF_WR_AB, wr_both) |
398 FIELD_PREP(OP_LDF_SRC_LMEXTN, src_lmextn) |
399 FIELD_PREP(OP_LDF_DST_LMEXTN, dst_lmextn);
400
401 nfp_prog_push(nfp_prog, insn);
402 }
403
404 static void
405 emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
406 enum shf_sc sc, u8 shift, bool zero)
407 {
408 struct nfp_insn_re_regs reg;
409 int err;
410
411 /* Note: ld_field is special as it uses one of the src regs as dst */
412 err = swreg_to_restricted(dst, dst, src, &reg, true);
413 if (err) {
414 nfp_prog->error = err;
415 return;
416 }
417
418 __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift,
419 reg.i8, zero, reg.swap, reg.wr_both,
420 reg.dst_lmextn, reg.src_lmextn);
421 }
422
423 static void
424 emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
425 enum shf_sc sc, u8 shift)
426 {
427 emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false);
428 }
429
430 static void emit_nop(struct nfp_prog *nfp_prog)
431 {
432 __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0);
433 }
434
435 /* --- Wrappers --- */
436 static bool pack_immed(u32 imm, u16 *val, enum immed_shift *shift)
437 {
438 if (!(imm & 0xffff0000)) {
439 *val = imm;
440 *shift = IMMED_SHIFT_0B;
441 } else if (!(imm & 0xff0000ff)) {
442 *val = imm >> 8;
443 *shift = IMMED_SHIFT_1B;
444 } else if (!(imm & 0x0000ffff)) {
445 *val = imm >> 16;
446 *shift = IMMED_SHIFT_2B;
447 } else {
448 return false;
449 }
450
451 return true;
452 }
453
454 static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm)
455 {
456 enum immed_shift shift;
457 u16 val;
458
459 if (pack_immed(imm, &val, &shift)) {
460 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift);
461 } else if (pack_immed(~imm, &val, &shift)) {
462 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift);
463 } else {
464 emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL,
465 false, IMMED_SHIFT_0B);
466 emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD,
467 false, IMMED_SHIFT_2B);
468 }
469 }
470
471 /* ur_load_imm_any() - encode immediate or use tmp register (unrestricted)
472 * If the @imm is small enough encode it directly in operand and return
473 * otherwise load @imm to a spare register and return its encoding.
474 */
475 static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
476 {
477 if (FIELD_FIT(UR_REG_IMM_MAX, imm))
478 return reg_imm(imm);
479
480 wrp_immed(nfp_prog, tmp_reg, imm);
481 return tmp_reg;
482 }
483
484 /* re_load_imm_any() - encode immediate or use tmp register (restricted)
485 * If the @imm is small enough encode it directly in operand and return
486 * otherwise load @imm to a spare register and return its encoding.
487 */
488 static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
489 {
490 if (FIELD_FIT(RE_REG_IMM_MAX, imm))
491 return reg_imm(imm);
492
493 wrp_immed(nfp_prog, tmp_reg, imm);
494 return tmp_reg;
495 }
496
497 static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count)
498 {
499 while (count--)
500 emit_nop(nfp_prog);
501 }
502
503 static void
504 wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask,
505 enum br_special special)
506 {
507 emit_br(nfp_prog, mask, 0, 0);
508
509 nfp_prog->prog[nfp_prog->prog_len - 1] |=
510 FIELD_PREP(OP_BR_SPECIAL, special);
511 }
512
513 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
514 {
515 emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
516 }
517
518 static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src)
519 {
520 wrp_mov(nfp_prog, reg_both(dst), reg_b(src));
521 }
522
523 static int
524 data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size)
525 {
526 unsigned int i;
527 u16 shift, sz;
528
529 /* We load the value from the address indicated in @offset and then
530 * shift out the data we don't need. Note: this is big endian!
531 */
532 sz = max(size, 4);
533 shift = size < 4 ? 4 - size : 0;
534
535 emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0,
536 pptr_reg(nfp_prog), offset, sz - 1, true);
537
538 i = 0;
539 if (shift)
540 emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE,
541 reg_xfer(0), SHF_SC_R_SHF, shift * 8);
542 else
543 for (; i * 4 < size; i++)
544 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
545
546 if (i < 2)
547 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
548
549 return 0;
550 }
551
552 static int
553 data_ld_host_order(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
554 u8 dst_gpr, int size)
555 {
556 unsigned int i;
557 u8 mask, sz;
558
559 /* We load the value from the address indicated in @offset and then
560 * mask out the data we don't need. Note: this is little endian!
561 */
562 sz = max(size, 4);
563 mask = size < 4 ? GENMASK(size - 1, 0) : 0;
564
565 emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0,
566 reg_a(src_gpr), offset, sz / 4 - 1, true);
567
568 i = 0;
569 if (mask)
570 emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask,
571 reg_xfer(0), SHF_SC_NONE, 0, true);
572 else
573 for (; i * 4 < size; i++)
574 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
575
576 if (i < 2)
577 wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
578
579 return 0;
580 }
581
582 static int
583 construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
584 {
585 swreg tmp_reg;
586
587 /* Calculate the true offset (src_reg + imm) */
588 tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
589 emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg);
590
591 /* Check packet length (size guaranteed to fit b/c it's u8) */
592 emit_alu(nfp_prog, imm_a(nfp_prog),
593 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
594 emit_alu(nfp_prog, reg_none(),
595 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));
596 wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
597
598 /* Load data */
599 return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
600 }
601
602 static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
603 {
604 swreg tmp_reg;
605
606 /* Check packet length */
607 tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));
608 emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
609 wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
610
611 /* Load data */
612 tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
613 return data_ld(nfp_prog, tmp_reg, 0, size);
614 }
615
616 static int
617 data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
618 u8 src_gpr, u8 size)
619 {
620 unsigned int i;
621
622 for (i = 0; i * 4 < size; i++)
623 wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i));
624
625 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
626 reg_a(dst_gpr), offset, size - 1, true);
627
628 return 0;
629 }
630
631 static int
632 data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
633 u64 imm, u8 size)
634 {
635 wrp_immed(nfp_prog, reg_xfer(0), imm);
636 if (size == 8)
637 wrp_immed(nfp_prog, reg_xfer(1), imm >> 32);
638
639 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
640 reg_a(dst_gpr), offset, size - 1, true);
641
642 return 0;
643 }
644
645 typedef int
646 (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off,
647 unsigned int size, bool first, bool new_gpr, bool last);
648
649 static int
650 wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off,
651 unsigned int size, bool first, bool new_gpr, bool last)
652 {
653 u32 idx, src_byte;
654 enum shf_sc sc;
655 swreg reg;
656 int shf;
657 u8 mask;
658
659 if (WARN_ON_ONCE(dst_byte + size > 4 || off % 4 + size > 4))
660 return -EOPNOTSUPP;
661
662 idx = off / 4;
663
664 /* Move the entire word */
665 if (size == 4) {
666 wrp_mov(nfp_prog, reg_both(dst), reg_lm(0, idx));
667 return 0;
668 }
669
670 src_byte = off % 4;
671
672 mask = (1 << size) - 1;
673 mask <<= dst_byte;
674
675 if (WARN_ON_ONCE(mask > 0xf))
676 return -EOPNOTSUPP;
677
678 shf = abs(src_byte - dst_byte) * 8;
679 if (src_byte == dst_byte) {
680 sc = SHF_SC_NONE;
681 } else if (src_byte < dst_byte) {
682 shf = 32 - shf;
683 sc = SHF_SC_L_SHF;
684 } else {
685 sc = SHF_SC_R_SHF;
686 }
687
688 /* ld_field can address fewer indexes, if offset too large do RMW.
689 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes.
690 */
691 if (idx <= RE_REG_LM_IDX_MAX) {
692 reg = reg_lm(0, idx);
693 } else {
694 reg = imm_a(nfp_prog);
695 /* If it's not the first part of the load and we start a new GPR
696 * that means we are loading a second part of the LMEM word into
697 * a new GPR. IOW we've already looked that LMEM word and
698 * therefore it has been loaded into imm_a().
699 */
700 if (first || !new_gpr)
701 wrp_mov(nfp_prog, reg, reg_lm(0, idx));
702 }
703
704 emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr);
705
706 return 0;
707 }
708
709 static int
710 wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off,
711 unsigned int size, bool first, bool new_gpr, bool last)
712 {
713 u32 idx, dst_byte;
714 enum shf_sc sc;
715 swreg reg;
716 int shf;
717 u8 mask;
718
719 if (WARN_ON_ONCE(src_byte + size > 4 || off % 4 + size > 4))
720 return -EOPNOTSUPP;
721
722 idx = off / 4;
723
724 /* Move the entire word */
725 if (size == 4) {
726 wrp_mov(nfp_prog, reg_lm(0, idx), reg_b(src));
727 return 0;
728 }
729
730 dst_byte = off % 4;
731
732 mask = (1 << size) - 1;
733 mask <<= dst_byte;
734
735 if (WARN_ON_ONCE(mask > 0xf))
736 return -EOPNOTSUPP;
737
738 shf = abs(src_byte - dst_byte) * 8;
739 if (src_byte == dst_byte) {
740 sc = SHF_SC_NONE;
741 } else if (src_byte < dst_byte) {
742 shf = 32 - shf;
743 sc = SHF_SC_L_SHF;
744 } else {
745 sc = SHF_SC_R_SHF;
746 }
747
748 /* ld_field can address fewer indexes, if offset too large do RMW.
749 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes.
750 */
751 if (idx <= RE_REG_LM_IDX_MAX) {
752 reg = reg_lm(0, idx);
753 } else {
754 reg = imm_a(nfp_prog);
755 /* Only first and last LMEM locations are going to need RMW,
756 * the middle location will be overwritten fully.
757 */
758 if (first || last)
759 wrp_mov(nfp_prog, reg, reg_lm(0, idx));
760 }
761
762 emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf);
763
764 if (new_gpr || last) {
765 if (idx > RE_REG_LM_IDX_MAX)
766 wrp_mov(nfp_prog, reg_lm(0, idx), reg);
767 }
768
769 return 0;
770 }
771
772 static int
773 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
774 unsigned int size, unsigned int ptr_off, u8 gpr, bool clr_gpr,
775 lmem_step step)
776 {
777 s32 off = nfp_prog->stack_depth + meta->insn.off + ptr_off;
778 bool first = true, last;
779 u8 prev_gpr = 255;
780 u32 gpr_byte = 0;
781 int ret;
782
783 if (clr_gpr && size < 8)
784 wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
785
786 while (size) {
787 u32 slice_end;
788 u8 slice_size;
789
790 slice_size = min(size, 4 - gpr_byte);
791 slice_end = min(off + slice_size, round_up(off + 1, 4));
792 slice_size = slice_end - off;
793
794 last = slice_size == size;
795
796 ret = step(nfp_prog, gpr, gpr_byte, off, slice_size,
797 first, gpr != prev_gpr, last);
798 if (ret)
799 return ret;
800
801 prev_gpr = gpr;
802 first = false;
803
804 gpr_byte += slice_size;
805 if (gpr_byte >= 4) {
806 gpr_byte -= 4;
807 gpr++;
808 }
809
810 size -= slice_size;
811 off += slice_size;
812 }
813
814 return 0;
815 }
816
817 static void
818 wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm)
819 {
820 swreg tmp_reg;
821
822 if (alu_op == ALU_OP_AND) {
823 if (!imm)
824 wrp_immed(nfp_prog, reg_both(dst), 0);
825 if (!imm || !~imm)
826 return;
827 }
828 if (alu_op == ALU_OP_OR) {
829 if (!~imm)
830 wrp_immed(nfp_prog, reg_both(dst), ~0U);
831 if (!imm || !~imm)
832 return;
833 }
834 if (alu_op == ALU_OP_XOR) {
835 if (!~imm)
836 emit_alu(nfp_prog, reg_both(dst), reg_none(),
837 ALU_OP_NEG, reg_b(dst));
838 if (!imm || !~imm)
839 return;
840 }
841
842 tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
843 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg);
844 }
845
846 static int
847 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
848 enum alu_op alu_op, bool skip)
849 {
850 const struct bpf_insn *insn = &meta->insn;
851 u64 imm = insn->imm; /* sign extend */
852
853 if (skip) {
854 meta->skip = true;
855 return 0;
856 }
857
858 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U);
859 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32);
860
861 return 0;
862 }
863
864 static int
865 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
866 enum alu_op alu_op)
867 {
868 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
869
870 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
871 emit_alu(nfp_prog, reg_both(dst + 1),
872 reg_a(dst + 1), alu_op, reg_b(src + 1));
873
874 return 0;
875 }
876
877 static int
878 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
879 enum alu_op alu_op, bool skip)
880 {
881 const struct bpf_insn *insn = &meta->insn;
882
883 if (skip) {
884 meta->skip = true;
885 return 0;
886 }
887
888 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm);
889 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
890
891 return 0;
892 }
893
894 static int
895 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
896 enum alu_op alu_op)
897 {
898 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
899
900 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
901 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
902
903 return 0;
904 }
905
906 static void
907 wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src,
908 enum br_mask br_mask, u16 off)
909 {
910 emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src));
911 emit_br(nfp_prog, br_mask, off, 0);
912 }
913
914 static int
915 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
916 enum alu_op alu_op, enum br_mask br_mask)
917 {
918 const struct bpf_insn *insn = &meta->insn;
919
920 if (insn->off < 0) /* TODO */
921 return -EOPNOTSUPP;
922
923 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op,
924 insn->src_reg * 2, br_mask, insn->off);
925 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
926 insn->src_reg * 2 + 1, br_mask, insn->off);
927
928 return 0;
929 }
930
931 static int
932 wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
933 enum br_mask br_mask, bool swap)
934 {
935 const struct bpf_insn *insn = &meta->insn;
936 u64 imm = insn->imm; /* sign extend */
937 u8 reg = insn->dst_reg * 2;
938 swreg tmp_reg;
939
940 if (insn->off < 0) /* TODO */
941 return -EOPNOTSUPP;
942
943 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
944 if (!swap)
945 emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg);
946 else
947 emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg));
948
949 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
950 if (!swap)
951 emit_alu(nfp_prog, reg_none(),
952 reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg);
953 else
954 emit_alu(nfp_prog, reg_none(),
955 tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1));
956
957 emit_br(nfp_prog, br_mask, insn->off, 0);
958
959 return 0;
960 }
961
962 static int
963 wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
964 enum br_mask br_mask, bool swap)
965 {
966 const struct bpf_insn *insn = &meta->insn;
967 u8 areg, breg;
968
969 areg = insn->dst_reg * 2;
970 breg = insn->src_reg * 2;
971
972 if (insn->off < 0) /* TODO */
973 return -EOPNOTSUPP;
974
975 if (swap) {
976 areg ^= breg;
977 breg ^= areg;
978 areg ^= breg;
979 }
980
981 emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg));
982 emit_alu(nfp_prog, reg_none(),
983 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1));
984 emit_br(nfp_prog, br_mask, insn->off, 0);
985
986 return 0;
987 }
988
989 static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out)
990 {
991 emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in,
992 SHF_SC_R_ROT, 8);
993 emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out),
994 SHF_SC_R_ROT, 16);
995 }
996
997 /* --- Callbacks --- */
998 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
999 {
1000 const struct bpf_insn *insn = &meta->insn;
1001
1002 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2);
1003 wrp_reg_mov(nfp_prog, insn->dst_reg * 2 + 1, insn->src_reg * 2 + 1);
1004
1005 return 0;
1006 }
1007
1008 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1009 {
1010 u64 imm = meta->insn.imm; /* sign extend */
1011
1012 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U);
1013 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32);
1014
1015 return 0;
1016 }
1017
1018 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1019 {
1020 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR);
1021 }
1022
1023 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1024 {
1025 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm);
1026 }
1027
1028 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1029 {
1030 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND);
1031 }
1032
1033 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1034 {
1035 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
1036 }
1037
1038 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1039 {
1040 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR);
1041 }
1042
1043 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1044 {
1045 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
1046 }
1047
1048 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1049 {
1050 const struct bpf_insn *insn = &meta->insn;
1051
1052 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
1053 reg_a(insn->dst_reg * 2), ALU_OP_ADD,
1054 reg_b(insn->src_reg * 2));
1055 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
1056 reg_a(insn->dst_reg * 2 + 1), ALU_OP_ADD_C,
1057 reg_b(insn->src_reg * 2 + 1));
1058
1059 return 0;
1060 }
1061
1062 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1063 {
1064 const struct bpf_insn *insn = &meta->insn;
1065 u64 imm = insn->imm; /* sign extend */
1066
1067 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U);
1068 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32);
1069
1070 return 0;
1071 }
1072
1073 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1074 {
1075 const struct bpf_insn *insn = &meta->insn;
1076
1077 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
1078 reg_a(insn->dst_reg * 2), ALU_OP_SUB,
1079 reg_b(insn->src_reg * 2));
1080 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
1081 reg_a(insn->dst_reg * 2 + 1), ALU_OP_SUB_C,
1082 reg_b(insn->src_reg * 2 + 1));
1083
1084 return 0;
1085 }
1086
1087 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1088 {
1089 const struct bpf_insn *insn = &meta->insn;
1090 u64 imm = insn->imm; /* sign extend */
1091
1092 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U);
1093 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32);
1094
1095 return 0;
1096 }
1097
1098 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1099 {
1100 const struct bpf_insn *insn = &meta->insn;
1101 u8 dst = insn->dst_reg * 2;
1102
1103 if (insn->imm < 32) {
1104 emit_shf(nfp_prog, reg_both(dst + 1),
1105 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst),
1106 SHF_SC_R_DSHF, 32 - insn->imm);
1107 emit_shf(nfp_prog, reg_both(dst),
1108 reg_none(), SHF_OP_NONE, reg_b(dst),
1109 SHF_SC_L_SHF, insn->imm);
1110 } else if (insn->imm == 32) {
1111 wrp_reg_mov(nfp_prog, dst + 1, dst);
1112 wrp_immed(nfp_prog, reg_both(dst), 0);
1113 } else if (insn->imm > 32) {
1114 emit_shf(nfp_prog, reg_both(dst + 1),
1115 reg_none(), SHF_OP_NONE, reg_b(dst),
1116 SHF_SC_L_SHF, insn->imm - 32);
1117 wrp_immed(nfp_prog, reg_both(dst), 0);
1118 }
1119
1120 return 0;
1121 }
1122
1123 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1124 {
1125 const struct bpf_insn *insn = &meta->insn;
1126 u8 dst = insn->dst_reg * 2;
1127
1128 if (insn->imm < 32) {
1129 emit_shf(nfp_prog, reg_both(dst),
1130 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst),
1131 SHF_SC_R_DSHF, insn->imm);
1132 emit_shf(nfp_prog, reg_both(dst + 1),
1133 reg_none(), SHF_OP_NONE, reg_b(dst + 1),
1134 SHF_SC_R_SHF, insn->imm);
1135 } else if (insn->imm == 32) {
1136 wrp_reg_mov(nfp_prog, dst, dst + 1);
1137 wrp_immed(nfp_prog, reg_both(dst + 1), 0);
1138 } else if (insn->imm > 32) {
1139 emit_shf(nfp_prog, reg_both(dst),
1140 reg_none(), SHF_OP_NONE, reg_b(dst + 1),
1141 SHF_SC_R_SHF, insn->imm - 32);
1142 wrp_immed(nfp_prog, reg_both(dst + 1), 0);
1143 }
1144
1145 return 0;
1146 }
1147
1148 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1149 {
1150 const struct bpf_insn *insn = &meta->insn;
1151
1152 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2);
1153 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
1154
1155 return 0;
1156 }
1157
1158 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1159 {
1160 const struct bpf_insn *insn = &meta->insn;
1161
1162 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
1163 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
1164
1165 return 0;
1166 }
1167
1168 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1169 {
1170 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR);
1171 }
1172
1173 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1174 {
1175 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm);
1176 }
1177
1178 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1179 {
1180 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND);
1181 }
1182
1183 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1184 {
1185 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
1186 }
1187
1188 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1189 {
1190 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR);
1191 }
1192
1193 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1194 {
1195 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
1196 }
1197
1198 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1199 {
1200 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD);
1201 }
1202
1203 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1204 {
1205 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm);
1206 }
1207
1208 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1209 {
1210 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB);
1211 }
1212
1213 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1214 {
1215 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm);
1216 }
1217
1218 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1219 {
1220 const struct bpf_insn *insn = &meta->insn;
1221
1222 if (!insn->imm)
1223 return 1; /* TODO: zero shift means indirect */
1224
1225 emit_shf(nfp_prog, reg_both(insn->dst_reg * 2),
1226 reg_none(), SHF_OP_NONE, reg_b(insn->dst_reg * 2),
1227 SHF_SC_L_SHF, insn->imm);
1228 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
1229
1230 return 0;
1231 }
1232
1233 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1234 {
1235 const struct bpf_insn *insn = &meta->insn;
1236 u8 gpr = insn->dst_reg * 2;
1237
1238 switch (insn->imm) {
1239 case 16:
1240 emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr),
1241 SHF_SC_R_ROT, 8);
1242 emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr),
1243 SHF_SC_R_SHF, 16);
1244
1245 wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
1246 break;
1247 case 32:
1248 wrp_end32(nfp_prog, reg_a(gpr), gpr);
1249 wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
1250 break;
1251 case 64:
1252 wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1));
1253
1254 wrp_end32(nfp_prog, reg_a(gpr), gpr + 1);
1255 wrp_end32(nfp_prog, imm_a(nfp_prog), gpr);
1256 break;
1257 }
1258
1259 return 0;
1260 }
1261
1262 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1263 {
1264 wrp_immed(nfp_prog, reg_both(nfp_meta_prev(meta)->insn.dst_reg * 2 + 1),
1265 meta->insn.imm);
1266
1267 return 0;
1268 }
1269
1270 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1271 {
1272 const struct bpf_insn *insn = &meta->insn;
1273
1274 meta->double_cb = imm_ld8_part2;
1275 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
1276
1277 return 0;
1278 }
1279
1280 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1281 {
1282 return construct_data_ld(nfp_prog, meta->insn.imm, 1);
1283 }
1284
1285 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1286 {
1287 return construct_data_ld(nfp_prog, meta->insn.imm, 2);
1288 }
1289
1290 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1291 {
1292 return construct_data_ld(nfp_prog, meta->insn.imm, 4);
1293 }
1294
1295 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1296 {
1297 return construct_data_ind_ld(nfp_prog, meta->insn.imm,
1298 meta->insn.src_reg * 2, 1);
1299 }
1300
1301 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1302 {
1303 return construct_data_ind_ld(nfp_prog, meta->insn.imm,
1304 meta->insn.src_reg * 2, 2);
1305 }
1306
1307 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1308 {
1309 return construct_data_ind_ld(nfp_prog, meta->insn.imm,
1310 meta->insn.src_reg * 2, 4);
1311 }
1312
1313 static int
1314 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1315 unsigned int size, unsigned int ptr_off)
1316 {
1317 return mem_op_stack(nfp_prog, meta, size, ptr_off,
1318 meta->insn.dst_reg * 2, true, wrp_lmem_load);
1319 }
1320
1321 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1322 u8 size)
1323 {
1324 swreg dst = reg_both(meta->insn.dst_reg * 2);
1325
1326 switch (meta->insn.off) {
1327 case offsetof(struct sk_buff, len):
1328 if (size != FIELD_SIZEOF(struct sk_buff, len))
1329 return -EOPNOTSUPP;
1330 wrp_mov(nfp_prog, dst, plen_reg(nfp_prog));
1331 break;
1332 case offsetof(struct sk_buff, data):
1333 if (size != sizeof(void *))
1334 return -EOPNOTSUPP;
1335 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
1336 break;
1337 case offsetof(struct sk_buff, cb) +
1338 offsetof(struct bpf_skb_data_end, data_end):
1339 if (size != sizeof(void *))
1340 return -EOPNOTSUPP;
1341 emit_alu(nfp_prog, dst,
1342 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
1343 break;
1344 default:
1345 return -EOPNOTSUPP;
1346 }
1347
1348 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
1349
1350 return 0;
1351 }
1352
1353 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1354 u8 size)
1355 {
1356 swreg dst = reg_both(meta->insn.dst_reg * 2);
1357
1358 if (size != sizeof(void *))
1359 return -EINVAL;
1360
1361 switch (meta->insn.off) {
1362 case offsetof(struct xdp_buff, data):
1363 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
1364 break;
1365 case offsetof(struct xdp_buff, data_end):
1366 emit_alu(nfp_prog, dst,
1367 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
1368 break;
1369 default:
1370 return -EOPNOTSUPP;
1371 }
1372
1373 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
1374
1375 return 0;
1376 }
1377
1378 static int
1379 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1380 unsigned int size)
1381 {
1382 swreg tmp_reg;
1383
1384 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
1385
1386 return data_ld_host_order(nfp_prog, meta->insn.src_reg * 2, tmp_reg,
1387 meta->insn.dst_reg * 2, size);
1388 }
1389
1390 static int
1391 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1392 unsigned int size)
1393 {
1394 if (meta->ptr.type == PTR_TO_CTX) {
1395 if (nfp_prog->act == NN_ACT_XDP)
1396 return mem_ldx_xdp(nfp_prog, meta, size);
1397 else
1398 return mem_ldx_skb(nfp_prog, meta, size);
1399 }
1400
1401 if (meta->ptr.type == PTR_TO_PACKET)
1402 return mem_ldx_data(nfp_prog, meta, size);
1403
1404 if (meta->ptr.type == PTR_TO_STACK)
1405 return mem_ldx_stack(nfp_prog, meta, size,
1406 meta->ptr.off + meta->ptr.var_off.value);
1407
1408 return -EOPNOTSUPP;
1409 }
1410
1411 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1412 {
1413 return mem_ldx(nfp_prog, meta, 1);
1414 }
1415
1416 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1417 {
1418 return mem_ldx(nfp_prog, meta, 2);
1419 }
1420
1421 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1422 {
1423 return mem_ldx(nfp_prog, meta, 4);
1424 }
1425
1426 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1427 {
1428 return mem_ldx(nfp_prog, meta, 8);
1429 }
1430
1431 static int
1432 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1433 unsigned int size)
1434 {
1435 u64 imm = meta->insn.imm; /* sign extend */
1436 swreg off_reg;
1437
1438 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
1439
1440 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
1441 imm, size);
1442 }
1443
1444 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1445 unsigned int size)
1446 {
1447 if (meta->ptr.type == PTR_TO_PACKET)
1448 return mem_st_data(nfp_prog, meta, size);
1449
1450 return -EOPNOTSUPP;
1451 }
1452
1453 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1454 {
1455 return mem_st(nfp_prog, meta, 1);
1456 }
1457
1458 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1459 {
1460 return mem_st(nfp_prog, meta, 2);
1461 }
1462
1463 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1464 {
1465 return mem_st(nfp_prog, meta, 4);
1466 }
1467
1468 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1469 {
1470 return mem_st(nfp_prog, meta, 8);
1471 }
1472
1473 static int
1474 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1475 unsigned int size)
1476 {
1477 swreg off_reg;
1478
1479 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
1480
1481 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
1482 meta->insn.src_reg * 2, size);
1483 }
1484
1485 static int
1486 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1487 unsigned int size, unsigned int ptr_off)
1488 {
1489 return mem_op_stack(nfp_prog, meta, size, ptr_off,
1490 meta->insn.src_reg * 2, false, wrp_lmem_store);
1491 }
1492
1493 static int
1494 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1495 unsigned int size)
1496 {
1497 if (meta->ptr.type == PTR_TO_PACKET)
1498 return mem_stx_data(nfp_prog, meta, size);
1499
1500 if (meta->ptr.type == PTR_TO_STACK)
1501 return mem_stx_stack(nfp_prog, meta, size,
1502 meta->ptr.off + meta->ptr.var_off.value);
1503
1504 return -EOPNOTSUPP;
1505 }
1506
1507 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1508 {
1509 return mem_stx(nfp_prog, meta, 1);
1510 }
1511
1512 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1513 {
1514 return mem_stx(nfp_prog, meta, 2);
1515 }
1516
1517 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1518 {
1519 return mem_stx(nfp_prog, meta, 4);
1520 }
1521
1522 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1523 {
1524 return mem_stx(nfp_prog, meta, 8);
1525 }
1526
1527 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1528 {
1529 if (meta->insn.off < 0) /* TODO */
1530 return -EOPNOTSUPP;
1531 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0);
1532
1533 return 0;
1534 }
1535
1536 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1537 {
1538 const struct bpf_insn *insn = &meta->insn;
1539 u64 imm = insn->imm; /* sign extend */
1540 swreg or1, or2, tmp_reg;
1541
1542 or1 = reg_a(insn->dst_reg * 2);
1543 or2 = reg_b(insn->dst_reg * 2 + 1);
1544
1545 if (insn->off < 0) /* TODO */
1546 return -EOPNOTSUPP;
1547
1548 if (imm & ~0U) {
1549 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
1550 emit_alu(nfp_prog, imm_a(nfp_prog),
1551 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
1552 or1 = imm_a(nfp_prog);
1553 }
1554
1555 if (imm >> 32) {
1556 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
1557 emit_alu(nfp_prog, imm_b(nfp_prog),
1558 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg);
1559 or2 = imm_b(nfp_prog);
1560 }
1561
1562 emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2);
1563 emit_br(nfp_prog, BR_BEQ, insn->off, 0);
1564
1565 return 0;
1566 }
1567
1568 static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1569 {
1570 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true);
1571 }
1572
1573 static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1574 {
1575 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false);
1576 }
1577
1578 static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1579 {
1580 return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false);
1581 }
1582
1583 static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1584 {
1585 return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true);
1586 }
1587
1588 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1589 {
1590 const struct bpf_insn *insn = &meta->insn;
1591 u64 imm = insn->imm; /* sign extend */
1592 swreg tmp_reg;
1593
1594 if (insn->off < 0) /* TODO */
1595 return -EOPNOTSUPP;
1596
1597 if (!imm) {
1598 meta->skip = true;
1599 return 0;
1600 }
1601
1602 if (imm & ~0U) {
1603 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
1604 emit_alu(nfp_prog, reg_none(),
1605 reg_a(insn->dst_reg * 2), ALU_OP_AND, tmp_reg);
1606 emit_br(nfp_prog, BR_BNE, insn->off, 0);
1607 }
1608
1609 if (imm >> 32) {
1610 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
1611 emit_alu(nfp_prog, reg_none(),
1612 reg_a(insn->dst_reg * 2 + 1), ALU_OP_AND, tmp_reg);
1613 emit_br(nfp_prog, BR_BNE, insn->off, 0);
1614 }
1615
1616 return 0;
1617 }
1618
1619 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1620 {
1621 const struct bpf_insn *insn = &meta->insn;
1622 u64 imm = insn->imm; /* sign extend */
1623 swreg tmp_reg;
1624
1625 if (insn->off < 0) /* TODO */
1626 return -EOPNOTSUPP;
1627
1628 if (!imm) {
1629 emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
1630 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1));
1631 emit_br(nfp_prog, BR_BNE, insn->off, 0);
1632 return 0;
1633 }
1634
1635 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
1636 emit_alu(nfp_prog, reg_none(),
1637 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
1638 emit_br(nfp_prog, BR_BNE, insn->off, 0);
1639
1640 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
1641 emit_alu(nfp_prog, reg_none(),
1642 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg);
1643 emit_br(nfp_prog, BR_BNE, insn->off, 0);
1644
1645 return 0;
1646 }
1647
1648 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1649 {
1650 const struct bpf_insn *insn = &meta->insn;
1651
1652 if (insn->off < 0) /* TODO */
1653 return -EOPNOTSUPP;
1654
1655 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2),
1656 ALU_OP_XOR, reg_b(insn->src_reg * 2));
1657 emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1),
1658 ALU_OP_XOR, reg_b(insn->src_reg * 2 + 1));
1659 emit_alu(nfp_prog, reg_none(),
1660 imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog));
1661 emit_br(nfp_prog, BR_BEQ, insn->off, 0);
1662
1663 return 0;
1664 }
1665
1666 static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1667 {
1668 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true);
1669 }
1670
1671 static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1672 {
1673 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false);
1674 }
1675
1676 static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1677 {
1678 return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false);
1679 }
1680
1681 static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1682 {
1683 return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true);
1684 }
1685
1686 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1687 {
1688 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
1689 }
1690
1691 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1692 {
1693 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
1694 }
1695
1696 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1697 {
1698 wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT);
1699
1700 return 0;
1701 }
1702
1703 static const instr_cb_t instr_cb[256] = {
1704 [BPF_ALU64 | BPF_MOV | BPF_X] = mov_reg64,
1705 [BPF_ALU64 | BPF_MOV | BPF_K] = mov_imm64,
1706 [BPF_ALU64 | BPF_XOR | BPF_X] = xor_reg64,
1707 [BPF_ALU64 | BPF_XOR | BPF_K] = xor_imm64,
1708 [BPF_ALU64 | BPF_AND | BPF_X] = and_reg64,
1709 [BPF_ALU64 | BPF_AND | BPF_K] = and_imm64,
1710 [BPF_ALU64 | BPF_OR | BPF_X] = or_reg64,
1711 [BPF_ALU64 | BPF_OR | BPF_K] = or_imm64,
1712 [BPF_ALU64 | BPF_ADD | BPF_X] = add_reg64,
1713 [BPF_ALU64 | BPF_ADD | BPF_K] = add_imm64,
1714 [BPF_ALU64 | BPF_SUB | BPF_X] = sub_reg64,
1715 [BPF_ALU64 | BPF_SUB | BPF_K] = sub_imm64,
1716 [BPF_ALU64 | BPF_LSH | BPF_K] = shl_imm64,
1717 [BPF_ALU64 | BPF_RSH | BPF_K] = shr_imm64,
1718 [BPF_ALU | BPF_MOV | BPF_X] = mov_reg,
1719 [BPF_ALU | BPF_MOV | BPF_K] = mov_imm,
1720 [BPF_ALU | BPF_XOR | BPF_X] = xor_reg,
1721 [BPF_ALU | BPF_XOR | BPF_K] = xor_imm,
1722 [BPF_ALU | BPF_AND | BPF_X] = and_reg,
1723 [BPF_ALU | BPF_AND | BPF_K] = and_imm,
1724 [BPF_ALU | BPF_OR | BPF_X] = or_reg,
1725 [BPF_ALU | BPF_OR | BPF_K] = or_imm,
1726 [BPF_ALU | BPF_ADD | BPF_X] = add_reg,
1727 [BPF_ALU | BPF_ADD | BPF_K] = add_imm,
1728 [BPF_ALU | BPF_SUB | BPF_X] = sub_reg,
1729 [BPF_ALU | BPF_SUB | BPF_K] = sub_imm,
1730 [BPF_ALU | BPF_LSH | BPF_K] = shl_imm,
1731 [BPF_ALU | BPF_END | BPF_X] = end_reg32,
1732 [BPF_LD | BPF_IMM | BPF_DW] = imm_ld8,
1733 [BPF_LD | BPF_ABS | BPF_B] = data_ld1,
1734 [BPF_LD | BPF_ABS | BPF_H] = data_ld2,
1735 [BPF_LD | BPF_ABS | BPF_W] = data_ld4,
1736 [BPF_LD | BPF_IND | BPF_B] = data_ind_ld1,
1737 [BPF_LD | BPF_IND | BPF_H] = data_ind_ld2,
1738 [BPF_LD | BPF_IND | BPF_W] = data_ind_ld4,
1739 [BPF_LDX | BPF_MEM | BPF_B] = mem_ldx1,
1740 [BPF_LDX | BPF_MEM | BPF_H] = mem_ldx2,
1741 [BPF_LDX | BPF_MEM | BPF_W] = mem_ldx4,
1742 [BPF_LDX | BPF_MEM | BPF_DW] = mem_ldx8,
1743 [BPF_STX | BPF_MEM | BPF_B] = mem_stx1,
1744 [BPF_STX | BPF_MEM | BPF_H] = mem_stx2,
1745 [BPF_STX | BPF_MEM | BPF_W] = mem_stx4,
1746 [BPF_STX | BPF_MEM | BPF_DW] = mem_stx8,
1747 [BPF_ST | BPF_MEM | BPF_B] = mem_st1,
1748 [BPF_ST | BPF_MEM | BPF_H] = mem_st2,
1749 [BPF_ST | BPF_MEM | BPF_W] = mem_st4,
1750 [BPF_ST | BPF_MEM | BPF_DW] = mem_st8,
1751 [BPF_JMP | BPF_JA | BPF_K] = jump,
1752 [BPF_JMP | BPF_JEQ | BPF_K] = jeq_imm,
1753 [BPF_JMP | BPF_JGT | BPF_K] = jgt_imm,
1754 [BPF_JMP | BPF_JGE | BPF_K] = jge_imm,
1755 [BPF_JMP | BPF_JLT | BPF_K] = jlt_imm,
1756 [BPF_JMP | BPF_JLE | BPF_K] = jle_imm,
1757 [BPF_JMP | BPF_JSET | BPF_K] = jset_imm,
1758 [BPF_JMP | BPF_JNE | BPF_K] = jne_imm,
1759 [BPF_JMP | BPF_JEQ | BPF_X] = jeq_reg,
1760 [BPF_JMP | BPF_JGT | BPF_X] = jgt_reg,
1761 [BPF_JMP | BPF_JGE | BPF_X] = jge_reg,
1762 [BPF_JMP | BPF_JLT | BPF_X] = jlt_reg,
1763 [BPF_JMP | BPF_JLE | BPF_X] = jle_reg,
1764 [BPF_JMP | BPF_JSET | BPF_X] = jset_reg,
1765 [BPF_JMP | BPF_JNE | BPF_X] = jne_reg,
1766 [BPF_JMP | BPF_EXIT] = goto_out,
1767 };
1768
1769 /* --- Misc code --- */
1770 static void br_set_offset(u64 *instr, u16 offset)
1771 {
1772 u16 addr_lo, addr_hi;
1773
1774 addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
1775 addr_hi = offset != addr_lo;
1776 *instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO);
1777 *instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
1778 *instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo);
1779 }
1780
1781 /* --- Assembler logic --- */
1782 static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
1783 {
1784 struct nfp_insn_meta *meta, *next;
1785 u32 off, br_idx;
1786 u32 idx;
1787
1788 nfp_for_each_insn_walk2(nfp_prog, meta, next) {
1789 if (meta->skip)
1790 continue;
1791 if (BPF_CLASS(meta->insn.code) != BPF_JMP)
1792 continue;
1793
1794 br_idx = nfp_prog_offset_to_index(nfp_prog, next->off) - 1;
1795 if (!nfp_is_br(nfp_prog->prog[br_idx])) {
1796 pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
1797 br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
1798 return -ELOOP;
1799 }
1800 /* Leave special branches for later */
1801 if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]))
1802 continue;
1803
1804 /* Find the target offset in assembler realm */
1805 off = meta->insn.off;
1806 if (!off) {
1807 pr_err("Fixup found zero offset!!\n");
1808 return -ELOOP;
1809 }
1810
1811 while (off && nfp_meta_has_next(nfp_prog, next)) {
1812 next = nfp_meta_next(next);
1813 off--;
1814 }
1815 if (off) {
1816 pr_err("Fixup found too large jump!! %d\n", off);
1817 return -ELOOP;
1818 }
1819
1820 if (next->skip) {
1821 pr_err("Branch landing on removed instruction!!\n");
1822 return -ELOOP;
1823 }
1824
1825 for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off);
1826 idx <= br_idx; idx++) {
1827 if (!nfp_is_br(nfp_prog->prog[idx]))
1828 continue;
1829 br_set_offset(&nfp_prog->prog[idx], next->off);
1830 }
1831 }
1832
1833 /* Fixup 'goto out's separately, they can be scattered around */
1834 for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) {
1835 enum br_special special;
1836
1837 if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE)
1838 continue;
1839
1840 special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]);
1841 switch (special) {
1842 case OP_BR_NORMAL:
1843 break;
1844 case OP_BR_GO_OUT:
1845 br_set_offset(&nfp_prog->prog[br_idx],
1846 nfp_prog->tgt_out);
1847 break;
1848 case OP_BR_GO_ABORT:
1849 br_set_offset(&nfp_prog->prog[br_idx],
1850 nfp_prog->tgt_abort);
1851 break;
1852 }
1853
1854 nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL;
1855 }
1856
1857 return 0;
1858 }
1859
1860 static void nfp_intro(struct nfp_prog *nfp_prog)
1861 {
1862 wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0));
1863 emit_alu(nfp_prog, plen_reg(nfp_prog),
1864 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog));
1865 }
1866
1867 static void nfp_outro_tc_legacy(struct nfp_prog *nfp_prog)
1868 {
1869 const u8 act2code[] = {
1870 [NN_ACT_TC_DROP] = 0x22,
1871 [NN_ACT_TC_REDIR] = 0x24
1872 };
1873 /* Target for aborts */
1874 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
1875 wrp_immed(nfp_prog, reg_both(0), 0);
1876
1877 /* Target for normal exits */
1878 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
1879 /* Legacy TC mode:
1880 * 0 0x11 -> pass, count as stat0
1881 * -1 drop 0x22 -> drop, count as stat1
1882 * redir 0x24 -> redir, count as stat1
1883 * ife mark 0x21 -> pass, count as stat1
1884 * ife + tx 0x24 -> redir, count as stat1
1885 */
1886 emit_br_byte_neq(nfp_prog, reg_b(0), 0xff, 0, nfp_prog->tgt_done, 2);
1887 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1888 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
1889
1890 emit_br(nfp_prog, BR_UNC, nfp_prog->tgt_done, 1);
1891 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(act2code[nfp_prog->act]),
1892 SHF_SC_L_SHF, 16);
1893 }
1894
1895 static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
1896 {
1897 /* TC direct-action mode:
1898 * 0,1 ok NOT SUPPORTED[1]
1899 * 2 drop 0x22 -> drop, count as stat1
1900 * 4,5 nuke 0x02 -> drop
1901 * 7 redir 0x44 -> redir, count as stat2
1902 * * unspec 0x11 -> pass, count as stat0
1903 *
1904 * [1] We can't support OK and RECLASSIFY because we can't tell TC
1905 * the exact decision made. We are forced to support UNSPEC
1906 * to handle aborts so that's the only one we handle for passing
1907 * packets up the stack.
1908 */
1909 /* Target for aborts */
1910 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
1911
1912 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
1913
1914 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1915 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
1916
1917 /* Target for normal exits */
1918 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
1919
1920 /* if R0 > 7 jump to abort */
1921 emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0));
1922 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
1923 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1924
1925 wrp_immed(nfp_prog, reg_b(2), 0x41221211);
1926 wrp_immed(nfp_prog, reg_b(3), 0x41001211);
1927
1928 emit_shf(nfp_prog, reg_a(1),
1929 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 2);
1930
1931 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
1932 emit_shf(nfp_prog, reg_a(2),
1933 reg_imm(0xf), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
1934
1935 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
1936 emit_shf(nfp_prog, reg_b(2),
1937 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0);
1938
1939 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
1940
1941 emit_shf(nfp_prog, reg_b(2),
1942 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4);
1943 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
1944 }
1945
1946 static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
1947 {
1948 /* XDP return codes:
1949 * 0 aborted 0x82 -> drop, count as stat3
1950 * 1 drop 0x22 -> drop, count as stat1
1951 * 2 pass 0x11 -> pass, count as stat0
1952 * 3 tx 0x44 -> redir, count as stat2
1953 * * unknown 0x82 -> drop, count as stat3
1954 */
1955 /* Target for aborts */
1956 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
1957
1958 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
1959
1960 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1961 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
1962
1963 /* Target for normal exits */
1964 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
1965
1966 /* if R0 > 3 jump to abort */
1967 emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0));
1968 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
1969
1970 wrp_immed(nfp_prog, reg_b(2), 0x44112282);
1971
1972 emit_shf(nfp_prog, reg_a(1),
1973 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 3);
1974
1975 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
1976 emit_shf(nfp_prog, reg_b(2),
1977 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
1978
1979 emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
1980
1981 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1982 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
1983 }
1984
1985 static void nfp_outro(struct nfp_prog *nfp_prog)
1986 {
1987 switch (nfp_prog->act) {
1988 case NN_ACT_DIRECT:
1989 nfp_outro_tc_da(nfp_prog);
1990 break;
1991 case NN_ACT_TC_DROP:
1992 case NN_ACT_TC_REDIR:
1993 nfp_outro_tc_legacy(nfp_prog);
1994 break;
1995 case NN_ACT_XDP:
1996 nfp_outro_xdp(nfp_prog);
1997 break;
1998 }
1999 }
2000
2001 static int nfp_translate(struct nfp_prog *nfp_prog)
2002 {
2003 struct nfp_insn_meta *meta;
2004 int err;
2005
2006 nfp_intro(nfp_prog);
2007 if (nfp_prog->error)
2008 return nfp_prog->error;
2009
2010 list_for_each_entry(meta, &nfp_prog->insns, l) {
2011 instr_cb_t cb = instr_cb[meta->insn.code];
2012
2013 meta->off = nfp_prog_current_offset(nfp_prog);
2014
2015 if (meta->skip) {
2016 nfp_prog->n_translated++;
2017 continue;
2018 }
2019
2020 if (nfp_meta_has_prev(nfp_prog, meta) &&
2021 nfp_meta_prev(meta)->double_cb)
2022 cb = nfp_meta_prev(meta)->double_cb;
2023 if (!cb)
2024 return -ENOENT;
2025 err = cb(nfp_prog, meta);
2026 if (err)
2027 return err;
2028
2029 nfp_prog->n_translated++;
2030 }
2031
2032 nfp_outro(nfp_prog);
2033 if (nfp_prog->error)
2034 return nfp_prog->error;
2035
2036 wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW);
2037 if (nfp_prog->error)
2038 return nfp_prog->error;
2039
2040 return nfp_fixup_branches(nfp_prog);
2041 }
2042
2043 static int
2044 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
2045 unsigned int cnt)
2046 {
2047 unsigned int i;
2048
2049 for (i = 0; i < cnt; i++) {
2050 struct nfp_insn_meta *meta;
2051
2052 meta = kzalloc(sizeof(*meta), GFP_KERNEL);
2053 if (!meta)
2054 return -ENOMEM;
2055
2056 meta->insn = prog[i];
2057 meta->n = i;
2058
2059 list_add_tail(&meta->l, &nfp_prog->insns);
2060 }
2061
2062 return 0;
2063 }
2064
2065 /* --- Optimizations --- */
2066 static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog)
2067 {
2068 struct nfp_insn_meta *meta;
2069
2070 list_for_each_entry(meta, &nfp_prog->insns, l) {
2071 struct bpf_insn insn = meta->insn;
2072
2073 /* Programs converted from cBPF start with register xoring */
2074 if (insn.code == (BPF_ALU64 | BPF_XOR | BPF_X) &&
2075 insn.src_reg == insn.dst_reg)
2076 continue;
2077
2078 /* Programs start with R6 = R1 but we ignore the skb pointer */
2079 if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) &&
2080 insn.src_reg == 1 && insn.dst_reg == 6)
2081 meta->skip = true;
2082
2083 /* Return as soon as something doesn't match */
2084 if (!meta->skip)
2085 return;
2086 }
2087 }
2088
2089 /* Remove masking after load since our load guarantees this is not needed */
2090 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
2091 {
2092 struct nfp_insn_meta *meta1, *meta2;
2093 const s32 exp_mask[] = {
2094 [BPF_B] = 0x000000ffU,
2095 [BPF_H] = 0x0000ffffU,
2096 [BPF_W] = 0xffffffffU,
2097 };
2098
2099 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
2100 struct bpf_insn insn, next;
2101
2102 insn = meta1->insn;
2103 next = meta2->insn;
2104
2105 if (BPF_CLASS(insn.code) != BPF_LD)
2106 continue;
2107 if (BPF_MODE(insn.code) != BPF_ABS &&
2108 BPF_MODE(insn.code) != BPF_IND)
2109 continue;
2110
2111 if (next.code != (BPF_ALU64 | BPF_AND | BPF_K))
2112 continue;
2113
2114 if (!exp_mask[BPF_SIZE(insn.code)])
2115 continue;
2116 if (exp_mask[BPF_SIZE(insn.code)] != next.imm)
2117 continue;
2118
2119 if (next.src_reg || next.dst_reg)
2120 continue;
2121
2122 meta2->skip = true;
2123 }
2124 }
2125
2126 static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog)
2127 {
2128 struct nfp_insn_meta *meta1, *meta2, *meta3;
2129
2130 nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) {
2131 struct bpf_insn insn, next1, next2;
2132
2133 insn = meta1->insn;
2134 next1 = meta2->insn;
2135 next2 = meta3->insn;
2136
2137 if (BPF_CLASS(insn.code) != BPF_LD)
2138 continue;
2139 if (BPF_MODE(insn.code) != BPF_ABS &&
2140 BPF_MODE(insn.code) != BPF_IND)
2141 continue;
2142 if (BPF_SIZE(insn.code) != BPF_W)
2143 continue;
2144
2145 if (!(next1.code == (BPF_LSH | BPF_K | BPF_ALU64) &&
2146 next2.code == (BPF_RSH | BPF_K | BPF_ALU64)) &&
2147 !(next1.code == (BPF_RSH | BPF_K | BPF_ALU64) &&
2148 next2.code == (BPF_LSH | BPF_K | BPF_ALU64)))
2149 continue;
2150
2151 if (next1.src_reg || next1.dst_reg ||
2152 next2.src_reg || next2.dst_reg)
2153 continue;
2154
2155 if (next1.imm != 0x20 || next2.imm != 0x20)
2156 continue;
2157
2158 meta2->skip = true;
2159 meta3->skip = true;
2160 }
2161 }
2162
2163 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
2164 {
2165 nfp_bpf_opt_reg_init(nfp_prog);
2166
2167 nfp_bpf_opt_ld_mask(nfp_prog);
2168 nfp_bpf_opt_ld_shift(nfp_prog);
2169
2170 return 0;
2171 }
2172
2173 static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore)
2174 {
2175 int i;
2176
2177 for (i = 0; i < nfp_prog->prog_len; i++) {
2178 int err;
2179
2180 err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]);
2181 if (err)
2182 return err;
2183
2184 nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]);
2185
2186 ustore[i] = cpu_to_le64(nfp_prog->prog[i]);
2187 }
2188
2189 return 0;
2190 }
2191
2192 /**
2193 * nfp_bpf_jit() - translate BPF code into NFP assembly
2194 * @filter: kernel BPF filter struct
2195 * @prog_mem: memory to store assembler instructions
2196 * @act: action attached to this eBPF program
2197 * @prog_start: offset of the first instruction when loaded
2198 * @prog_done: where to jump on exit
2199 * @prog_sz: size of @prog_mem in instructions
2200 * @res: achieved parameters of translation results
2201 */
2202 int
2203 nfp_bpf_jit(struct bpf_prog *filter, void *prog_mem,
2204 enum nfp_bpf_action_type act,
2205 unsigned int prog_start, unsigned int prog_done,
2206 unsigned int prog_sz, struct nfp_bpf_result *res)
2207 {
2208 struct nfp_prog *nfp_prog;
2209 int ret;
2210
2211 nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
2212 if (!nfp_prog)
2213 return -ENOMEM;
2214
2215 INIT_LIST_HEAD(&nfp_prog->insns);
2216 nfp_prog->act = act;
2217 nfp_prog->start_off = prog_start;
2218 nfp_prog->tgt_done = prog_done;
2219
2220 ret = nfp_prog_prepare(nfp_prog, filter->insnsi, filter->len);
2221 if (ret)
2222 goto out;
2223
2224 ret = nfp_prog_verify(nfp_prog, filter);
2225 if (ret)
2226 goto out;
2227
2228 ret = nfp_bpf_optimize(nfp_prog);
2229 if (ret)
2230 goto out;
2231
2232 nfp_prog->num_regs = MAX_BPF_REG;
2233 nfp_prog->regs_per_thread = 32;
2234
2235 nfp_prog->prog = prog_mem;
2236 nfp_prog->__prog_alloc_len = prog_sz;
2237
2238 ret = nfp_translate(nfp_prog);
2239 if (ret) {
2240 pr_err("Translation failed with error %d (translated: %u)\n",
2241 ret, nfp_prog->n_translated);
2242 ret = -EINVAL;
2243 goto out;
2244 }
2245
2246 ret = nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)prog_mem);
2247
2248 res->n_instr = nfp_prog->prog_len;
2249 res->dense_mode = false;
2250 out:
2251 nfp_prog_free(nfp_prog);
2252
2253 return ret;
2254 }
(deprecated) Btrfs devel tree