1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
6 #include <linux/module.h>
7 #include <linux/init.h>
8 #include <linux/kernel.h>
9 #include <linux/skbuff.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/filter.h>
12 #include <linux/bpf.h>
14 #include <net/netlink.h>
15 #include <net/pkt_sched.h>
16 #include <net/pkt_cls.h>
18 #include <linux/tc_act/tc_bpf.h>
19 #include <net/tc_act/tc_bpf.h>
21 #define ACT_BPF_NAME_LEN 256
24 struct bpf_prog
*filter
;
25 struct sock_filter
*bpf_ops
;
31 static unsigned int bpf_net_id
;
32 static struct tc_action_ops act_bpf_ops
;
34 static int tcf_bpf_act(struct sk_buff
*skb
, const struct tc_action
*act
,
35 struct tcf_result
*res
)
37 bool at_ingress
= skb_at_tc_ingress(skb
);
38 struct tcf_bpf
*prog
= to_bpf(act
);
39 struct bpf_prog
*filter
;
40 int action
, filter_res
;
42 tcf_lastuse_update(&prog
->tcf_tm
);
43 bstats_cpu_update(this_cpu_ptr(prog
->common
.cpu_bstats
), skb
);
46 filter
= rcu_dereference(prog
->filter
);
48 __skb_push(skb
, skb
->mac_len
);
49 bpf_compute_data_pointers(skb
);
50 filter_res
= BPF_PROG_RUN(filter
, skb
);
51 __skb_pull(skb
, skb
->mac_len
);
53 bpf_compute_data_pointers(skb
);
54 filter_res
= BPF_PROG_RUN(filter
, skb
);
58 /* A BPF program may overwrite the default action opcode.
59 * Similarly as in cls_bpf, if filter_res == -1 we use the
60 * default action specified from tc.
62 * In case a different well-known TC_ACT opcode has been
63 * returned, it will overwrite the default one.
65 * For everything else that is unkown, TC_ACT_UNSPEC is
70 case TC_ACT_RECLASSIFY
:
77 qstats_drop_inc(this_cpu_ptr(prog
->common
.cpu_qstats
));
80 action
= prog
->tcf_action
;
83 action
= TC_ACT_UNSPEC
;
90 static bool tcf_bpf_is_ebpf(const struct tcf_bpf
*prog
)
92 return !prog
->bpf_ops
;
95 static int tcf_bpf_dump_bpf_info(const struct tcf_bpf
*prog
,
100 if (nla_put_u16(skb
, TCA_ACT_BPF_OPS_LEN
, prog
->bpf_num_ops
))
103 nla
= nla_reserve(skb
, TCA_ACT_BPF_OPS
, prog
->bpf_num_ops
*
104 sizeof(struct sock_filter
));
108 memcpy(nla_data(nla
), prog
->bpf_ops
, nla_len(nla
));
113 static int tcf_bpf_dump_ebpf_info(const struct tcf_bpf
*prog
,
118 if (prog
->bpf_name
&&
119 nla_put_string(skb
, TCA_ACT_BPF_NAME
, prog
->bpf_name
))
122 if (nla_put_u32(skb
, TCA_ACT_BPF_ID
, prog
->filter
->aux
->id
))
125 nla
= nla_reserve(skb
, TCA_ACT_BPF_TAG
, sizeof(prog
->filter
->tag
));
129 memcpy(nla_data(nla
), prog
->filter
->tag
, nla_len(nla
));
134 static int tcf_bpf_dump(struct sk_buff
*skb
, struct tc_action
*act
,
137 unsigned char *tp
= skb_tail_pointer(skb
);
138 struct tcf_bpf
*prog
= to_bpf(act
);
139 struct tc_act_bpf opt
= {
140 .index
= prog
->tcf_index
,
141 .refcnt
= refcount_read(&prog
->tcf_refcnt
) - ref
,
142 .bindcnt
= atomic_read(&prog
->tcf_bindcnt
) - bind
,
147 spin_lock_bh(&prog
->tcf_lock
);
148 opt
.action
= prog
->tcf_action
;
149 if (nla_put(skb
, TCA_ACT_BPF_PARMS
, sizeof(opt
), &opt
))
150 goto nla_put_failure
;
152 if (tcf_bpf_is_ebpf(prog
))
153 ret
= tcf_bpf_dump_ebpf_info(prog
, skb
);
155 ret
= tcf_bpf_dump_bpf_info(prog
, skb
);
157 goto nla_put_failure
;
159 tcf_tm_dump(&tm
, &prog
->tcf_tm
);
160 if (nla_put_64bit(skb
, TCA_ACT_BPF_TM
, sizeof(tm
), &tm
,
162 goto nla_put_failure
;
164 spin_unlock_bh(&prog
->tcf_lock
);
168 spin_unlock_bh(&prog
->tcf_lock
);
173 static const struct nla_policy act_bpf_policy
[TCA_ACT_BPF_MAX
+ 1] = {
174 [TCA_ACT_BPF_PARMS
] = { .len
= sizeof(struct tc_act_bpf
) },
175 [TCA_ACT_BPF_FD
] = { .type
= NLA_U32
},
176 [TCA_ACT_BPF_NAME
] = { .type
= NLA_NUL_STRING
,
177 .len
= ACT_BPF_NAME_LEN
},
178 [TCA_ACT_BPF_OPS_LEN
] = { .type
= NLA_U16
},
179 [TCA_ACT_BPF_OPS
] = { .type
= NLA_BINARY
,
180 .len
= sizeof(struct sock_filter
) * BPF_MAXINSNS
},
183 static int tcf_bpf_init_from_ops(struct nlattr
**tb
, struct tcf_bpf_cfg
*cfg
)
185 struct sock_filter
*bpf_ops
;
186 struct sock_fprog_kern fprog_tmp
;
188 u16 bpf_size
, bpf_num_ops
;
191 bpf_num_ops
= nla_get_u16(tb
[TCA_ACT_BPF_OPS_LEN
]);
192 if (bpf_num_ops
> BPF_MAXINSNS
|| bpf_num_ops
== 0)
195 bpf_size
= bpf_num_ops
* sizeof(*bpf_ops
);
196 if (bpf_size
!= nla_len(tb
[TCA_ACT_BPF_OPS
]))
199 bpf_ops
= kmemdup(nla_data(tb
[TCA_ACT_BPF_OPS
]), bpf_size
, GFP_KERNEL
);
203 fprog_tmp
.len
= bpf_num_ops
;
204 fprog_tmp
.filter
= bpf_ops
;
206 ret
= bpf_prog_create(&fp
, &fprog_tmp
);
212 cfg
->bpf_ops
= bpf_ops
;
213 cfg
->bpf_num_ops
= bpf_num_ops
;
215 cfg
->is_ebpf
= false;
220 static int tcf_bpf_init_from_efd(struct nlattr
**tb
, struct tcf_bpf_cfg
*cfg
)
226 bpf_fd
= nla_get_u32(tb
[TCA_ACT_BPF_FD
]);
228 fp
= bpf_prog_get_type(bpf_fd
, BPF_PROG_TYPE_SCHED_ACT
);
232 if (tb
[TCA_ACT_BPF_NAME
]) {
233 name
= nla_memdup(tb
[TCA_ACT_BPF_NAME
], GFP_KERNEL
);
240 cfg
->bpf_name
= name
;
247 static void tcf_bpf_cfg_cleanup(const struct tcf_bpf_cfg
*cfg
)
249 struct bpf_prog
*filter
= cfg
->filter
;
253 bpf_prog_put(filter
);
255 bpf_prog_destroy(filter
);
259 kfree(cfg
->bpf_name
);
262 static void tcf_bpf_prog_fill_cfg(const struct tcf_bpf
*prog
,
263 struct tcf_bpf_cfg
*cfg
)
265 cfg
->is_ebpf
= tcf_bpf_is_ebpf(prog
);
266 /* updates to prog->filter are prevented, since it's called either
267 * with tcf lock or during final cleanup in rcu callback
269 cfg
->filter
= rcu_dereference_protected(prog
->filter
, 1);
271 cfg
->bpf_ops
= prog
->bpf_ops
;
272 cfg
->bpf_name
= prog
->bpf_name
;
275 static int tcf_bpf_init(struct net
*net
, struct nlattr
*nla
,
276 struct nlattr
*est
, struct tc_action
**act
,
277 int replace
, int bind
, bool rtnl_held
,
278 struct tcf_proto
*tp
, u32 flags
,
279 struct netlink_ext_ack
*extack
)
281 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
282 struct nlattr
*tb
[TCA_ACT_BPF_MAX
+ 1];
283 struct tcf_chain
*goto_ch
= NULL
;
284 struct tcf_bpf_cfg cfg
, old
;
285 struct tc_act_bpf
*parm
;
286 struct tcf_bpf
*prog
;
287 bool is_bpf
, is_ebpf
;
294 ret
= nla_parse_nested_deprecated(tb
, TCA_ACT_BPF_MAX
, nla
,
295 act_bpf_policy
, NULL
);
299 if (!tb
[TCA_ACT_BPF_PARMS
])
302 parm
= nla_data(tb
[TCA_ACT_BPF_PARMS
]);
304 ret
= tcf_idr_check_alloc(tn
, &index
, act
, bind
);
306 ret
= tcf_idr_create(tn
, index
, est
, act
,
307 &act_bpf_ops
, bind
, true, 0);
309 tcf_idr_cleanup(tn
, index
);
314 } else if (ret
> 0) {
315 /* Don't override defaults. */
320 tcf_idr_release(*act
, bind
);
327 ret
= tcf_action_check_ctrlact(parm
->action
, tp
, &goto_ch
, extack
);
331 is_bpf
= tb
[TCA_ACT_BPF_OPS_LEN
] && tb
[TCA_ACT_BPF_OPS
];
332 is_ebpf
= tb
[TCA_ACT_BPF_FD
];
334 if ((!is_bpf
&& !is_ebpf
) || (is_bpf
&& is_ebpf
)) {
339 memset(&cfg
, 0, sizeof(cfg
));
341 ret
= is_bpf
? tcf_bpf_init_from_ops(tb
, &cfg
) :
342 tcf_bpf_init_from_efd(tb
, &cfg
);
348 spin_lock_bh(&prog
->tcf_lock
);
349 if (res
!= ACT_P_CREATED
)
350 tcf_bpf_prog_fill_cfg(prog
, &old
);
352 prog
->bpf_ops
= cfg
.bpf_ops
;
353 prog
->bpf_name
= cfg
.bpf_name
;
356 prog
->bpf_num_ops
= cfg
.bpf_num_ops
;
358 goto_ch
= tcf_action_set_ctrlact(*act
, parm
->action
, goto_ch
);
359 rcu_assign_pointer(prog
->filter
, cfg
.filter
);
360 spin_unlock_bh(&prog
->tcf_lock
);
363 tcf_chain_put_by_act(goto_ch
);
365 if (res
== ACT_P_CREATED
) {
366 tcf_idr_insert(tn
, *act
);
368 /* make sure the program being replaced is no longer executing */
370 tcf_bpf_cfg_cleanup(&old
);
377 tcf_chain_put_by_act(goto_ch
);
380 tcf_idr_release(*act
, bind
);
384 static void tcf_bpf_cleanup(struct tc_action
*act
)
386 struct tcf_bpf_cfg tmp
;
388 tcf_bpf_prog_fill_cfg(to_bpf(act
), &tmp
);
389 tcf_bpf_cfg_cleanup(&tmp
);
392 static int tcf_bpf_walker(struct net
*net
, struct sk_buff
*skb
,
393 struct netlink_callback
*cb
, int type
,
394 const struct tc_action_ops
*ops
,
395 struct netlink_ext_ack
*extack
)
397 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
399 return tcf_generic_walker(tn
, skb
, cb
, type
, ops
, extack
);
402 static int tcf_bpf_search(struct net
*net
, struct tc_action
**a
, u32 index
)
404 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
406 return tcf_idr_search(tn
, a
, index
);
409 static struct tc_action_ops act_bpf_ops __read_mostly
= {
412 .owner
= THIS_MODULE
,
414 .dump
= tcf_bpf_dump
,
415 .cleanup
= tcf_bpf_cleanup
,
416 .init
= tcf_bpf_init
,
417 .walk
= tcf_bpf_walker
,
418 .lookup
= tcf_bpf_search
,
419 .size
= sizeof(struct tcf_bpf
),
422 static __net_init
int bpf_init_net(struct net
*net
)
424 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
426 return tc_action_net_init(net
, tn
, &act_bpf_ops
);
429 static void __net_exit
bpf_exit_net(struct list_head
*net_list
)
431 tc_action_net_exit(net_list
, bpf_net_id
);
434 static struct pernet_operations bpf_net_ops
= {
435 .init
= bpf_init_net
,
436 .exit_batch
= bpf_exit_net
,
438 .size
= sizeof(struct tc_action_net
),
441 static int __init
bpf_init_module(void)
443 return tcf_register_action(&act_bpf_ops
, &bpf_net_ops
);
446 static void __exit
bpf_cleanup_module(void)
448 tcf_unregister_action(&act_bpf_ops
, &bpf_net_ops
);
451 module_init(bpf_init_module
);
452 module_exit(bpf_cleanup_module
);
454 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
455 MODULE_DESCRIPTION("TC BPF based action");
456 MODULE_LICENSE("GPL v2");