split dev_queue

[cor.git] / kernel / bpf / helpers.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 */
#include <linux/bpf.h>
#include <linux/rcupdate.h>
#include <linux/random.h>
#include <linux/smp.h>
#include <linux/topology.h>
#include <linux/ktime.h>
#include <linux/sched.h>
#include <linux/uidgid.h>
#include <linux/filter.h>
#include <linux/ctype.h>

#include "../../lib/kstrtox.h"

/* If kernel subsystem is allowing eBPF programs to call this function,
 * inside its own verifier_ops->get_func_proto() callback it should return
 * bpf_map_lookup_elem_proto, so that verifier can properly check the arguments
 *
 * Different map implementations will rely on rcu in map methods
 * lookup/update/delete, therefore eBPF programs must run under rcu lock
 * if program is allowed to access maps, so check rcu_read_lock_held in
 * all three functions.
 */
BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
{
        WARN_ON_ONCE(!rcu_read_lock_held());
        return (unsigned long) map->ops->map_lookup_elem(map, key);
}

const struct bpf_func_proto bpf_map_lookup_elem_proto = {
        .func           = bpf_map_lookup_elem,
        .gpl_only       = false,
        .pkt_access     = true,
        .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
        .arg1_type      = ARG_CONST_MAP_PTR,
        .arg2_type      = ARG_PTR_TO_MAP_KEY,
};

BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
           void *, value, u64, flags)
{
        WARN_ON_ONCE(!rcu_read_lock_held());
        return map->ops->map_update_elem(map, key, value, flags);
}

const struct bpf_func_proto bpf_map_update_elem_proto = {
        .func           = bpf_map_update_elem,
        .gpl_only       = false,
        .pkt_access     = true,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_CONST_MAP_PTR,
        .arg2_type      = ARG_PTR_TO_MAP_KEY,
        .arg3_type      = ARG_PTR_TO_MAP_VALUE,
        .arg4_type      = ARG_ANYTHING,
};

BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
{
        WARN_ON_ONCE(!rcu_read_lock_held());
        return map->ops->map_delete_elem(map, key);
}

const struct bpf_func_proto bpf_map_delete_elem_proto = {
        .func           = bpf_map_delete_elem,
        .gpl_only       = false,
        .pkt_access     = true,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_CONST_MAP_PTR,
        .arg2_type      = ARG_PTR_TO_MAP_KEY,
};

BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags)
{
        return map->ops->map_push_elem(map, value, flags);
}

const struct bpf_func_proto bpf_map_push_elem_proto = {
        .func           = bpf_map_push_elem,
        .gpl_only       = false,
        .pkt_access     = true,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_CONST_MAP_PTR,
        .arg2_type      = ARG_PTR_TO_MAP_VALUE,
        .arg3_type      = ARG_ANYTHING,
};

BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value)
{
        return map->ops->map_pop_elem(map, value);
}

const struct bpf_func_proto bpf_map_pop_elem_proto = {
        .func           = bpf_map_pop_elem,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_CONST_MAP_PTR,
        .arg2_type      = ARG_PTR_TO_UNINIT_MAP_VALUE,
};

BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value)
{
        return map->ops->map_peek_elem(map, value);
}

const struct bpf_func_proto bpf_map_peek_elem_proto = {
        .func           = bpf_map_pop_elem,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_CONST_MAP_PTR,
        .arg2_type      = ARG_PTR_TO_UNINIT_MAP_VALUE,
};

const struct bpf_func_proto bpf_get_prandom_u32_proto = {
        .func           = bpf_user_rnd_u32,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
};

BPF_CALL_0(bpf_get_smp_processor_id)
{
        return smp_processor_id();
}

const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
        .func           = bpf_get_smp_processor_id,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
};

BPF_CALL_0(bpf_get_numa_node_id)
{
        return numa_node_id();
}

const struct bpf_func_proto bpf_get_numa_node_id_proto = {
        .func           = bpf_get_numa_node_id,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
};

BPF_CALL_0(bpf_ktime_get_ns)
{
        /* NMI safe access to clock monotonic */
        return ktime_get_mono_fast_ns();
}

const struct bpf_func_proto bpf_ktime_get_ns_proto = {
        .func           = bpf_ktime_get_ns,
        .gpl_only       = true,
        .ret_type       = RET_INTEGER,
};

BPF_CALL_0(bpf_get_current_pid_tgid)
{
        struct task_struct *task = current;

        if (unlikely(!task))
                return -EINVAL;

        return (u64) task->tgid << 32 | task->pid;
}

const struct bpf_func_proto bpf_get_current_pid_tgid_proto = {
        .func           = bpf_get_current_pid_tgid,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
};

BPF_CALL_0(bpf_get_current_uid_gid)
{
        struct task_struct *task = current;
        kuid_t uid;
        kgid_t gid;

        if (unlikely(!task))
                return -EINVAL;

        current_uid_gid(&uid, &gid);
        return (u64) from_kgid(&init_user_ns, gid) << 32 |
                     from_kuid(&init_user_ns, uid);
}

const struct bpf_func_proto bpf_get_current_uid_gid_proto = {
        .func           = bpf_get_current_uid_gid,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
};

BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size)
{
        struct task_struct *task = current;

        if (unlikely(!task))
                goto err_clear;

        strncpy(buf, task->comm, size);

        /* Verifier guarantees that size > 0. For task->comm exceeding
         * size, guarantee that buf is %NUL-terminated. Unconditionally
         * done here to save the size test.
         */
        buf[size - 1] = 0;
        return 0;
err_clear:
        memset(buf, 0, size);
        return -EINVAL;
}

const struct bpf_func_proto bpf_get_current_comm_proto = {
        .func           = bpf_get_current_comm,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_UNINIT_MEM,
        .arg2_type      = ARG_CONST_SIZE,
};

#if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK)

static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
{
        arch_spinlock_t *l = (void *)lock;
        union {
                __u32 val;
                arch_spinlock_t lock;
        } u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED };

        compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0");
        BUILD_BUG_ON(sizeof(*l) != sizeof(__u32));
        BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32));
        arch_spin_lock(l);
}

static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
{
        arch_spinlock_t *l = (void *)lock;

        arch_spin_unlock(l);
}

#else

static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
{
        atomic_t *l = (void *)lock;

        BUILD_BUG_ON(sizeof(*l) != sizeof(*lock));
        do {
                atomic_cond_read_relaxed(l, !VAL);
        } while (atomic_xchg(l, 1));
}

static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
{
        atomic_t *l = (void *)lock;

        atomic_set_release(l, 0);
}

#endif

static DEFINE_PER_CPU(unsigned long, irqsave_flags);

notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)
{
        unsigned long flags;

        local_irq_save(flags);
        __bpf_spin_lock(lock);
        __this_cpu_write(irqsave_flags, flags);
        return 0;
}

const struct bpf_func_proto bpf_spin_lock_proto = {
        .func           = bpf_spin_lock,
        .gpl_only       = false,
        .ret_type       = RET_VOID,
        .arg1_type      = ARG_PTR_TO_SPIN_LOCK,
};

notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
{
        unsigned long flags;

        flags = __this_cpu_read(irqsave_flags);
        __bpf_spin_unlock(lock);
        local_irq_restore(flags);
        return 0;
}

const struct bpf_func_proto bpf_spin_unlock_proto = {
        .func           = bpf_spin_unlock,
        .gpl_only       = false,
        .ret_type       = RET_VOID,
        .arg1_type      = ARG_PTR_TO_SPIN_LOCK,
};

void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
                           bool lock_src)
{
        struct bpf_spin_lock *lock;

        if (lock_src)
                lock = src + map->spin_lock_off;
        else
                lock = dst + map->spin_lock_off;
        preempt_disable();
        ____bpf_spin_lock(lock);
        copy_map_value(map, dst, src);
        ____bpf_spin_unlock(lock);
        preempt_enable();
}

#ifdef CONFIG_CGROUPS
BPF_CALL_0(bpf_get_current_cgroup_id)
{
        struct cgroup *cgrp = task_dfl_cgroup(current);

        return cgroup_id(cgrp);
}

const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
        .func           = bpf_get_current_cgroup_id,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
};

#ifdef CONFIG_CGROUP_BPF
DECLARE_PER_CPU(struct bpf_cgroup_storage*,
                bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);

BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
{
        /* flags argument is not used now,
         * but provides an ability to extend the API.
         * verifier checks that its value is correct.
         */
        enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
        struct bpf_cgroup_storage *storage;
        void *ptr;

        storage = this_cpu_read(bpf_cgroup_storage[stype]);

        if (stype == BPF_CGROUP_STORAGE_SHARED)
                ptr = &READ_ONCE(storage->buf)->data[0];
        else
                ptr = this_cpu_ptr(storage->percpu_buf);

        return (unsigned long)ptr;
}

const struct bpf_func_proto bpf_get_local_storage_proto = {
        .func           = bpf_get_local_storage,
        .gpl_only       = false,
        .ret_type       = RET_PTR_TO_MAP_VALUE,
        .arg1_type      = ARG_CONST_MAP_PTR,
        .arg2_type      = ARG_ANYTHING,
};
#endif

#define BPF_STRTOX_BASE_MASK 0x1F

static int __bpf_strtoull(const char *buf, size_t buf_len, u64 flags,
                          unsigned long long *res, bool *is_negative)
{
        unsigned int base = flags & BPF_STRTOX_BASE_MASK;
        const char *cur_buf = buf;
        size_t cur_len = buf_len;
        unsigned int consumed;
        size_t val_len;
        char str[64];

        if (!buf || !buf_len || !res || !is_negative)
                return -EINVAL;

        if (base != 0 && base != 8 && base != 10 && base != 16)
                return -EINVAL;

        if (flags & ~BPF_STRTOX_BASE_MASK)
                return -EINVAL;

        while (cur_buf < buf + buf_len && isspace(*cur_buf))
                ++cur_buf;

        *is_negative = (cur_buf < buf + buf_len && *cur_buf == '-');
        if (*is_negative)
                ++cur_buf;

        consumed = cur_buf - buf;
        cur_len -= consumed;
        if (!cur_len)
                return -EINVAL;

        cur_len = min(cur_len, sizeof(str) - 1);
        memcpy(str, cur_buf, cur_len);
        str[cur_len] = '\0';
        cur_buf = str;

        cur_buf = _parse_integer_fixup_radix(cur_buf, &base);
        val_len = _parse_integer(cur_buf, base, res);

        if (val_len & KSTRTOX_OVERFLOW)
                return -ERANGE;

        if (val_len == 0)
                return -EINVAL;

        cur_buf += val_len;
        consumed += cur_buf - str;

        return consumed;
}

static int __bpf_strtoll(const char *buf, size_t buf_len, u64 flags,
                         long long *res)
{
        unsigned long long _res;
        bool is_negative;
        int err;

        err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative);
        if (err < 0)
                return err;
        if (is_negative) {
                if ((long long)-_res > 0)
                        return -ERANGE;
                *res = -_res;
        } else {
                if ((long long)_res < 0)
                        return -ERANGE;
                *res = _res;
        }
        return err;
}

BPF_CALL_4(bpf_strtol, const char *, buf, size_t, buf_len, u64, flags,
           long *, res)
{
        long long _res;
        int err;

        err = __bpf_strtoll(buf, buf_len, flags, &_res);
        if (err < 0)
                return err;
        if (_res != (long)_res)
                return -ERANGE;
        *res = _res;
        return err;
}

const struct bpf_func_proto bpf_strtol_proto = {
        .func           = bpf_strtol,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_MEM,
        .arg2_type      = ARG_CONST_SIZE,
        .arg3_type      = ARG_ANYTHING,
        .arg4_type      = ARG_PTR_TO_LONG,
};

BPF_CALL_4(bpf_strtoul, const char *, buf, size_t, buf_len, u64, flags,
           unsigned long *, res)
{
        unsigned long long _res;
        bool is_negative;
        int err;

        err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative);
        if (err < 0)
                return err;
        if (is_negative)
                return -EINVAL;
        if (_res != (unsigned long)_res)
                return -ERANGE;
        *res = _res;
        return err;
}

const struct bpf_func_proto bpf_strtoul_proto = {
        .func           = bpf_strtoul,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_MEM,
        .arg2_type      = ARG_CONST_SIZE,
        .arg3_type      = ARG_ANYTHING,
        .arg4_type      = ARG_PTR_TO_LONG,
};
#endif