1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
10 #include <uapi/linux/bpf.h>
12 #include <linux/workqueue.h>
13 #include <linux/file.h>
14 #include <linux/percpu.h>
15 #include <linux/err.h>
16 #include <linux/rbtree_latch.h>
17 #include <linux/numa.h>
18 #include <linux/wait.h>
20 struct bpf_verifier_env
;
28 /* map is generic key/value storage optionally accesible by eBPF programs */
30 /* funcs callable from userspace (via syscall) */
31 int (*map_alloc_check
)(union bpf_attr
*attr
);
32 struct bpf_map
*(*map_alloc
)(union bpf_attr
*attr
);
33 void (*map_release
)(struct bpf_map
*map
, struct file
*map_file
);
34 void (*map_free
)(struct bpf_map
*map
);
35 int (*map_get_next_key
)(struct bpf_map
*map
, void *key
, void *next_key
);
36 void (*map_release_uref
)(struct bpf_map
*map
);
38 /* funcs callable from userspace and from eBPF programs */
39 void *(*map_lookup_elem
)(struct bpf_map
*map
, void *key
);
40 int (*map_update_elem
)(struct bpf_map
*map
, void *key
, void *value
, u64 flags
);
41 int (*map_delete_elem
)(struct bpf_map
*map
, void *key
);
43 /* funcs called by prog_array and perf_event_array map */
44 void *(*map_fd_get_ptr
)(struct bpf_map
*map
, struct file
*map_file
,
46 void (*map_fd_put_ptr
)(void *ptr
);
47 u32 (*map_gen_lookup
)(struct bpf_map
*map
, struct bpf_insn
*insn_buf
);
48 u32 (*map_fd_sys_lookup_elem
)(void *ptr
);
49 void (*map_seq_show_elem
)(struct bpf_map
*map
, void *key
,
51 int (*map_check_btf
)(const struct bpf_map
*map
, const struct btf
*btf
,
52 u32 key_type_id
, u32 value_type_id
);
56 /* The first two cachelines with read-mostly members of which some
57 * are also accessed in fast-path (e.g. ops, max_entries).
59 const struct bpf_map_ops
*ops ____cacheline_aligned
;
60 struct bpf_map
*inner_map_meta
;
61 #ifdef CONFIG_SECURITY
64 enum bpf_map_type map_type
;
73 u32 btf_value_type_id
;
78 /* The 3rd and 4th cacheline with misc members to avoid false sharing
79 * particularly with refcounting.
81 struct user_struct
*user ____cacheline_aligned
;
84 struct work_struct work
;
85 char name
[BPF_OBJ_NAME_LEN
];
88 struct bpf_offloaded_map
;
90 struct bpf_map_dev_ops
{
91 int (*map_get_next_key
)(struct bpf_offloaded_map
*map
,
92 void *key
, void *next_key
);
93 int (*map_lookup_elem
)(struct bpf_offloaded_map
*map
,
94 void *key
, void *value
);
95 int (*map_update_elem
)(struct bpf_offloaded_map
*map
,
96 void *key
, void *value
, u64 flags
);
97 int (*map_delete_elem
)(struct bpf_offloaded_map
*map
, void *key
);
100 struct bpf_offloaded_map
{
102 struct net_device
*netdev
;
103 const struct bpf_map_dev_ops
*dev_ops
;
105 struct list_head offloads
;
108 static inline struct bpf_offloaded_map
*map_to_offmap(struct bpf_map
*map
)
110 return container_of(map
, struct bpf_offloaded_map
, map
);
113 static inline bool bpf_map_offload_neutral(const struct bpf_map
*map
)
115 return map
->map_type
== BPF_MAP_TYPE_PERF_EVENT_ARRAY
;
118 static inline bool bpf_map_support_seq_show(const struct bpf_map
*map
)
120 return map
->ops
->map_seq_show_elem
&& map
->ops
->map_check_btf
;
123 extern const struct bpf_map_ops bpf_map_offload_ops
;
125 /* function argument constraints */
127 ARG_DONTCARE
= 0, /* unused argument in helper function */
129 /* the following constraints used to prototype
130 * bpf_map_lookup/update/delete_elem() functions
132 ARG_CONST_MAP_PTR
, /* const argument used as pointer to bpf_map */
133 ARG_PTR_TO_MAP_KEY
, /* pointer to stack used as map key */
134 ARG_PTR_TO_MAP_VALUE
, /* pointer to stack used as map value */
136 /* the following constraints used to prototype bpf_memcmp() and other
137 * functions that access data on eBPF program stack
139 ARG_PTR_TO_MEM
, /* pointer to valid memory (stack, packet, map value) */
140 ARG_PTR_TO_MEM_OR_NULL
, /* pointer to valid memory or NULL */
141 ARG_PTR_TO_UNINIT_MEM
, /* pointer to memory does not need to be initialized,
142 * helper function must fill all bytes or clear
143 * them in error case.
146 ARG_CONST_SIZE
, /* number of bytes accessed from memory */
147 ARG_CONST_SIZE_OR_ZERO
, /* number of bytes accessed from memory or 0 */
149 ARG_PTR_TO_CTX
, /* pointer to context */
150 ARG_ANYTHING
, /* any (initialized) argument is ok */
153 /* type of values returned from helper functions */
154 enum bpf_return_type
{
155 RET_INTEGER
, /* function returns integer */
156 RET_VOID
, /* function doesn't return anything */
157 RET_PTR_TO_MAP_VALUE_OR_NULL
, /* returns a pointer to map elem value or NULL */
160 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
161 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
162 * instructions after verifying
164 struct bpf_func_proto
{
165 u64 (*func
)(u64 r1
, u64 r2
, u64 r3
, u64 r4
, u64 r5
);
168 enum bpf_return_type ret_type
;
169 enum bpf_arg_type arg1_type
;
170 enum bpf_arg_type arg2_type
;
171 enum bpf_arg_type arg3_type
;
172 enum bpf_arg_type arg4_type
;
173 enum bpf_arg_type arg5_type
;
176 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is
177 * the first argument to eBPF programs.
178 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
182 enum bpf_access_type
{
187 /* types of values stored in eBPF registers */
188 /* Pointer types represent:
191 * pointer + (u16) var
192 * pointer + (u16) var + imm
193 * if (range > 0) then [ptr, ptr + range - off) is safe to access
194 * if (id > 0) means that some 'var' was added
195 * if (off > 0) means that 'imm' was added
198 NOT_INIT
= 0, /* nothing was written into register */
199 SCALAR_VALUE
, /* reg doesn't contain a valid pointer */
200 PTR_TO_CTX
, /* reg points to bpf_context */
201 CONST_PTR_TO_MAP
, /* reg points to struct bpf_map */
202 PTR_TO_MAP_VALUE
, /* reg points to map element value */
203 PTR_TO_MAP_VALUE_OR_NULL
,/* points to map elem value or NULL */
204 PTR_TO_STACK
, /* reg == frame_pointer + offset */
205 PTR_TO_PACKET_META
, /* skb->data - meta_len */
206 PTR_TO_PACKET
, /* reg points to skb->data */
207 PTR_TO_PACKET_END
, /* skb->data + headlen */
210 /* The information passed from prog-specific *_is_valid_access
211 * back to the verifier.
213 struct bpf_insn_access_aux
{
214 enum bpf_reg_type reg_type
;
219 bpf_ctx_record_field_size(struct bpf_insn_access_aux
*aux
, u32 size
)
221 aux
->ctx_field_size
= size
;
224 struct bpf_prog_ops
{
225 int (*test_run
)(struct bpf_prog
*prog
, const union bpf_attr
*kattr
,
226 union bpf_attr __user
*uattr
);
229 struct bpf_verifier_ops
{
230 /* return eBPF function prototype for verification */
231 const struct bpf_func_proto
*
232 (*get_func_proto
)(enum bpf_func_id func_id
,
233 const struct bpf_prog
*prog
);
235 /* return true if 'size' wide access at offset 'off' within bpf_context
236 * with 'type' (read or write) is allowed
238 bool (*is_valid_access
)(int off
, int size
, enum bpf_access_type type
,
239 const struct bpf_prog
*prog
,
240 struct bpf_insn_access_aux
*info
);
241 int (*gen_prologue
)(struct bpf_insn
*insn
, bool direct_write
,
242 const struct bpf_prog
*prog
);
243 int (*gen_ld_abs
)(const struct bpf_insn
*orig
,
244 struct bpf_insn
*insn_buf
);
245 u32 (*convert_ctx_access
)(enum bpf_access_type type
,
246 const struct bpf_insn
*src
,
247 struct bpf_insn
*dst
,
248 struct bpf_prog
*prog
, u32
*target_size
);
251 struct bpf_prog_offload_ops
{
252 int (*insn_hook
)(struct bpf_verifier_env
*env
,
253 int insn_idx
, int prev_insn_idx
);
256 struct bpf_prog_offload
{
257 struct bpf_prog
*prog
;
258 struct net_device
*netdev
;
260 struct list_head offloads
;
262 const struct bpf_prog_offload_ops
*dev_ops
;
267 struct bpf_prog_aux
{
274 bool offload_requested
;
275 struct bpf_prog
**func
;
276 void *jit_data
; /* JIT specific data. arch dependent */
277 struct latch_tree_node ksym_tnode
;
278 struct list_head ksym_lnode
;
279 const struct bpf_prog_ops
*ops
;
280 struct bpf_map
**used_maps
;
281 struct bpf_prog
*prog
;
282 struct user_struct
*user
;
283 u64 load_time
; /* ns since boottime */
284 char name
[BPF_OBJ_NAME_LEN
];
285 #ifdef CONFIG_SECURITY
288 struct bpf_prog_offload
*offload
;
290 struct work_struct work
;
299 /* 'ownership' of prog_array is claimed by the first program that
300 * is going to use this map or by the first program which FD is stored
301 * in the map to make sure that all callers and callees have the same
302 * prog_type and JITed flag
304 enum bpf_prog_type owner_prog_type
;
307 char value
[0] __aligned(8);
308 void *ptrs
[0] __aligned(8);
309 void __percpu
*pptrs
[0] __aligned(8);
313 #define MAX_TAIL_CALL_CNT 32
315 struct bpf_event_entry
{
316 struct perf_event
*event
;
317 struct file
*perf_file
;
318 struct file
*map_file
;
322 bool bpf_prog_array_compatible(struct bpf_array
*array
, const struct bpf_prog
*fp
);
323 int bpf_prog_calc_tag(struct bpf_prog
*fp
);
325 const struct bpf_func_proto
*bpf_get_trace_printk_proto(void);
327 typedef unsigned long (*bpf_ctx_copy_t
)(void *dst
, const void *src
,
328 unsigned long off
, unsigned long len
);
330 u64
bpf_event_output(struct bpf_map
*map
, u64 flags
, void *meta
, u64 meta_size
,
331 void *ctx
, u64 ctx_size
, bpf_ctx_copy_t ctx_copy
);
333 int bpf_prog_test_run_xdp(struct bpf_prog
*prog
, const union bpf_attr
*kattr
,
334 union bpf_attr __user
*uattr
);
335 int bpf_prog_test_run_skb(struct bpf_prog
*prog
, const union bpf_attr
*kattr
,
336 union bpf_attr __user
*uattr
);
338 /* an array of programs to be executed under rcu_lock.
341 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN);
343 * the structure returned by bpf_prog_array_alloc() should be populated
344 * with program pointers and the last pointer must be NULL.
345 * The user has to keep refcnt on the program and make sure the program
346 * is removed from the array before bpf_prog_put().
347 * The 'struct bpf_prog_array *' should only be replaced with xchg()
348 * since other cpus are walking the array of pointers in parallel.
350 struct bpf_prog_array
{
352 struct bpf_prog
*progs
[0];
355 struct bpf_prog_array __rcu
*bpf_prog_array_alloc(u32 prog_cnt
, gfp_t flags
);
356 void bpf_prog_array_free(struct bpf_prog_array __rcu
*progs
);
357 int bpf_prog_array_length(struct bpf_prog_array __rcu
*progs
);
358 int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu
*progs
,
359 __u32 __user
*prog_ids
, u32 cnt
);
361 void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu
*progs
,
362 struct bpf_prog
*old_prog
);
363 int bpf_prog_array_copy_info(struct bpf_prog_array __rcu
*array
,
364 u32
*prog_ids
, u32 request_cnt
,
366 int bpf_prog_array_copy(struct bpf_prog_array __rcu
*old_array
,
367 struct bpf_prog
*exclude_prog
,
368 struct bpf_prog
*include_prog
,
369 struct bpf_prog_array
**new_array
);
371 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
373 struct bpf_prog **_prog, *__prog; \
374 struct bpf_prog_array *_array; \
378 _array = rcu_dereference(array); \
379 if (unlikely(check_non_null && !_array))\
381 _prog = _array->progs; \
382 while ((__prog = READ_ONCE(*_prog))) { \
383 _ret &= func(__prog, ctx); \
388 preempt_enable_no_resched(); \
392 #define BPF_PROG_RUN_ARRAY(array, ctx, func) \
393 __BPF_PROG_RUN_ARRAY(array, ctx, func, false)
395 #define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \
396 __BPF_PROG_RUN_ARRAY(array, ctx, func, true)
398 #ifdef CONFIG_BPF_SYSCALL
399 DECLARE_PER_CPU(int, bpf_prog_active
);
401 extern const struct file_operations bpf_map_fops
;
402 extern const struct file_operations bpf_prog_fops
;
404 #define BPF_PROG_TYPE(_id, _name) \
405 extern const struct bpf_prog_ops _name ## _prog_ops; \
406 extern const struct bpf_verifier_ops _name ## _verifier_ops;
407 #define BPF_MAP_TYPE(_id, _ops) \
408 extern const struct bpf_map_ops _ops;
409 #include <linux/bpf_types.h>
413 extern const struct bpf_prog_ops bpf_offload_prog_ops
;
414 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops
;
415 extern const struct bpf_verifier_ops xdp_analyzer_ops
;
417 struct bpf_prog
*bpf_prog_get(u32 ufd
);
418 struct bpf_prog
*bpf_prog_get_type_dev(u32 ufd
, enum bpf_prog_type type
,
420 struct bpf_prog
* __must_check
bpf_prog_add(struct bpf_prog
*prog
, int i
);
421 void bpf_prog_sub(struct bpf_prog
*prog
, int i
);
422 struct bpf_prog
* __must_check
bpf_prog_inc(struct bpf_prog
*prog
);
423 struct bpf_prog
* __must_check
bpf_prog_inc_not_zero(struct bpf_prog
*prog
);
424 void bpf_prog_put(struct bpf_prog
*prog
);
425 int __bpf_prog_charge(struct user_struct
*user
, u32 pages
);
426 void __bpf_prog_uncharge(struct user_struct
*user
, u32 pages
);
428 void bpf_prog_free_id(struct bpf_prog
*prog
, bool do_idr_lock
);
429 void bpf_map_free_id(struct bpf_map
*map
, bool do_idr_lock
);
431 struct bpf_map
*bpf_map_get_with_uref(u32 ufd
);
432 struct bpf_map
*__bpf_map_get(struct fd f
);
433 struct bpf_map
* __must_check
bpf_map_inc(struct bpf_map
*map
, bool uref
);
434 void bpf_map_put_with_uref(struct bpf_map
*map
);
435 void bpf_map_put(struct bpf_map
*map
);
436 int bpf_map_precharge_memlock(u32 pages
);
437 void *bpf_map_area_alloc(size_t size
, int numa_node
);
438 void bpf_map_area_free(void *base
);
439 void bpf_map_init_from_attr(struct bpf_map
*map
, union bpf_attr
*attr
);
441 extern int sysctl_unprivileged_bpf_disabled
;
443 int bpf_map_new_fd(struct bpf_map
*map
, int flags
);
444 int bpf_prog_new_fd(struct bpf_prog
*prog
);
446 int bpf_obj_pin_user(u32 ufd
, const char __user
*pathname
);
447 int bpf_obj_get_user(const char __user
*pathname
, int flags
);
449 int bpf_percpu_hash_copy(struct bpf_map
*map
, void *key
, void *value
);
450 int bpf_percpu_array_copy(struct bpf_map
*map
, void *key
, void *value
);
451 int bpf_percpu_hash_update(struct bpf_map
*map
, void *key
, void *value
,
453 int bpf_percpu_array_update(struct bpf_map
*map
, void *key
, void *value
,
456 int bpf_stackmap_copy(struct bpf_map
*map
, void *key
, void *value
);
458 int bpf_fd_array_map_update_elem(struct bpf_map
*map
, struct file
*map_file
,
459 void *key
, void *value
, u64 map_flags
);
460 int bpf_fd_array_map_lookup_elem(struct bpf_map
*map
, void *key
, u32
*value
);
461 int bpf_fd_htab_map_update_elem(struct bpf_map
*map
, struct file
*map_file
,
462 void *key
, void *value
, u64 map_flags
);
463 int bpf_fd_htab_map_lookup_elem(struct bpf_map
*map
, void *key
, u32
*value
);
465 int bpf_get_file_flag(int flags
);
466 int bpf_check_uarg_tail_zero(void __user
*uaddr
, size_t expected_size
,
469 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
470 * forced to use 'long' read/writes to try to atomically copy long counters.
471 * Best-effort only. No barriers here, since it _will_ race with concurrent
472 * updates from BPF programs. Called from bpf syscall and mostly used with
473 * size 8 or 16 bytes, so ask compiler to inline it.
475 static inline void bpf_long_memcpy(void *dst
, const void *src
, u32 size
)
477 const long *lsrc
= src
;
480 size
/= sizeof(long);
485 /* verify correctness of eBPF program */
486 int bpf_check(struct bpf_prog
**fp
, union bpf_attr
*attr
);
487 void bpf_patch_call_args(struct bpf_insn
*insn
, u32 stack_depth
);
493 struct bpf_dtab_netdev
*__dev_map_lookup_elem(struct bpf_map
*map
, u32 key
);
494 void __dev_map_insert_ctx(struct bpf_map
*map
, u32 index
);
495 void __dev_map_flush(struct bpf_map
*map
);
496 int dev_map_enqueue(struct bpf_dtab_netdev
*dst
, struct xdp_buff
*xdp
,
497 struct net_device
*dev_rx
);
498 int dev_map_generic_redirect(struct bpf_dtab_netdev
*dst
, struct sk_buff
*skb
,
499 struct bpf_prog
*xdp_prog
);
501 struct bpf_cpu_map_entry
*__cpu_map_lookup_elem(struct bpf_map
*map
, u32 key
);
502 void __cpu_map_insert_ctx(struct bpf_map
*map
, u32 index
);
503 void __cpu_map_flush(struct bpf_map
*map
);
504 int cpu_map_enqueue(struct bpf_cpu_map_entry
*rcpu
, struct xdp_buff
*xdp
,
505 struct net_device
*dev_rx
);
507 /* Return map's numa specified by userspace */
508 static inline int bpf_map_attr_numa_node(const union bpf_attr
*attr
)
510 return (attr
->map_flags
& BPF_F_NUMA_NODE
) ?
511 attr
->numa_node
: NUMA_NO_NODE
;
514 struct bpf_prog
*bpf_prog_get_type_path(const char *name
, enum bpf_prog_type type
);
516 #else /* !CONFIG_BPF_SYSCALL */
517 static inline struct bpf_prog
*bpf_prog_get(u32 ufd
)
519 return ERR_PTR(-EOPNOTSUPP
);
522 static inline struct bpf_prog
*bpf_prog_get_type_dev(u32 ufd
,
523 enum bpf_prog_type type
,
526 return ERR_PTR(-EOPNOTSUPP
);
529 static inline struct bpf_prog
* __must_check
bpf_prog_add(struct bpf_prog
*prog
,
532 return ERR_PTR(-EOPNOTSUPP
);
535 static inline void bpf_prog_sub(struct bpf_prog
*prog
, int i
)
539 static inline void bpf_prog_put(struct bpf_prog
*prog
)
543 static inline struct bpf_prog
* __must_check
bpf_prog_inc(struct bpf_prog
*prog
)
545 return ERR_PTR(-EOPNOTSUPP
);
548 static inline struct bpf_prog
*__must_check
549 bpf_prog_inc_not_zero(struct bpf_prog
*prog
)
551 return ERR_PTR(-EOPNOTSUPP
);
554 static inline int __bpf_prog_charge(struct user_struct
*user
, u32 pages
)
559 static inline void __bpf_prog_uncharge(struct user_struct
*user
, u32 pages
)
563 static inline int bpf_obj_get_user(const char __user
*pathname
, int flags
)
568 static inline struct net_device
*__dev_map_lookup_elem(struct bpf_map
*map
,
574 static inline void __dev_map_insert_ctx(struct bpf_map
*map
, u32 index
)
578 static inline void __dev_map_flush(struct bpf_map
*map
)
583 struct bpf_dtab_netdev
;
586 int dev_map_enqueue(struct bpf_dtab_netdev
*dst
, struct xdp_buff
*xdp
,
587 struct net_device
*dev_rx
)
594 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev
*dst
,
596 struct bpf_prog
*xdp_prog
)
602 struct bpf_cpu_map_entry
*__cpu_map_lookup_elem(struct bpf_map
*map
, u32 key
)
607 static inline void __cpu_map_insert_ctx(struct bpf_map
*map
, u32 index
)
611 static inline void __cpu_map_flush(struct bpf_map
*map
)
615 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry
*rcpu
,
616 struct xdp_buff
*xdp
,
617 struct net_device
*dev_rx
)
622 static inline struct bpf_prog
*bpf_prog_get_type_path(const char *name
,
623 enum bpf_prog_type type
)
625 return ERR_PTR(-EOPNOTSUPP
);
627 #endif /* CONFIG_BPF_SYSCALL */
629 static inline struct bpf_prog
*bpf_prog_get_type(u32 ufd
,
630 enum bpf_prog_type type
)
632 return bpf_prog_get_type_dev(ufd
, type
, false);
635 bool bpf_prog_get_ok(struct bpf_prog
*, enum bpf_prog_type
*, bool);
637 int bpf_prog_offload_compile(struct bpf_prog
*prog
);
638 void bpf_prog_offload_destroy(struct bpf_prog
*prog
);
639 int bpf_prog_offload_info_fill(struct bpf_prog_info
*info
,
640 struct bpf_prog
*prog
);
642 int bpf_map_offload_info_fill(struct bpf_map_info
*info
, struct bpf_map
*map
);
644 int bpf_map_offload_lookup_elem(struct bpf_map
*map
, void *key
, void *value
);
645 int bpf_map_offload_update_elem(struct bpf_map
*map
,
646 void *key
, void *value
, u64 flags
);
647 int bpf_map_offload_delete_elem(struct bpf_map
*map
, void *key
);
648 int bpf_map_offload_get_next_key(struct bpf_map
*map
,
649 void *key
, void *next_key
);
651 bool bpf_offload_dev_match(struct bpf_prog
*prog
, struct bpf_map
*map
);
653 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
654 int bpf_prog_offload_init(struct bpf_prog
*prog
, union bpf_attr
*attr
);
656 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux
*aux
)
658 return aux
->offload_requested
;
661 static inline bool bpf_map_is_dev_bound(struct bpf_map
*map
)
663 return unlikely(map
->ops
== &bpf_map_offload_ops
);
666 struct bpf_map
*bpf_map_offload_map_alloc(union bpf_attr
*attr
);
667 void bpf_map_offload_map_free(struct bpf_map
*map
);
669 static inline int bpf_prog_offload_init(struct bpf_prog
*prog
,
670 union bpf_attr
*attr
)
675 static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux
*aux
)
680 static inline bool bpf_map_is_dev_bound(struct bpf_map
*map
)
685 static inline struct bpf_map
*bpf_map_offload_map_alloc(union bpf_attr
*attr
)
687 return ERR_PTR(-EOPNOTSUPP
);
690 static inline void bpf_map_offload_map_free(struct bpf_map
*map
)
693 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
695 #if defined(CONFIG_STREAM_PARSER) && defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_INET)
696 struct sock
*__sock_map_lookup_elem(struct bpf_map
*map
, u32 key
);
697 struct sock
*__sock_hash_lookup_elem(struct bpf_map
*map
, void *key
);
698 int sock_map_prog(struct bpf_map
*map
, struct bpf_prog
*prog
, u32 type
);
699 int sockmap_get_from_fd(const union bpf_attr
*attr
, int type
,
700 struct bpf_prog
*prog
);
702 static inline struct sock
*__sock_map_lookup_elem(struct bpf_map
*map
, u32 key
)
707 static inline struct sock
*__sock_hash_lookup_elem(struct bpf_map
*map
,
713 static inline int sock_map_prog(struct bpf_map
*map
,
714 struct bpf_prog
*prog
,
720 static inline int sockmap_get_from_fd(const union bpf_attr
*attr
, int type
,
721 struct bpf_prog
*prog
)
727 #if defined(CONFIG_XDP_SOCKETS)
729 struct xdp_sock
*__xsk_map_lookup_elem(struct bpf_map
*map
, u32 key
);
730 int __xsk_map_redirect(struct bpf_map
*map
, struct xdp_buff
*xdp
,
731 struct xdp_sock
*xs
);
732 void __xsk_map_flush(struct bpf_map
*map
);
735 static inline struct xdp_sock
*__xsk_map_lookup_elem(struct bpf_map
*map
,
741 static inline int __xsk_map_redirect(struct bpf_map
*map
, struct xdp_buff
*xdp
,
747 static inline void __xsk_map_flush(struct bpf_map
*map
)
752 /* verifier prototypes for helper functions called from eBPF programs */
753 extern const struct bpf_func_proto bpf_map_lookup_elem_proto
;
754 extern const struct bpf_func_proto bpf_map_update_elem_proto
;
755 extern const struct bpf_func_proto bpf_map_delete_elem_proto
;
757 extern const struct bpf_func_proto bpf_get_prandom_u32_proto
;
758 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto
;
759 extern const struct bpf_func_proto bpf_get_numa_node_id_proto
;
760 extern const struct bpf_func_proto bpf_tail_call_proto
;
761 extern const struct bpf_func_proto bpf_ktime_get_ns_proto
;
762 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto
;
763 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto
;
764 extern const struct bpf_func_proto bpf_get_current_comm_proto
;
765 extern const struct bpf_func_proto bpf_get_stackid_proto
;
766 extern const struct bpf_func_proto bpf_get_stack_proto
;
767 extern const struct bpf_func_proto bpf_sock_map_update_proto
;
768 extern const struct bpf_func_proto bpf_sock_hash_update_proto
;
769 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto
;
771 /* Shared helpers among cBPF and eBPF. */
772 void bpf_user_rnd_init_once(void);
773 u64
bpf_user_rnd_u32(u64 r1
, u64 r2
, u64 r3
, u64 r4
, u64 r5
);
775 #endif /* _LINUX_BPF_H */