AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Browse Source 
Daniel Borkmann says:

====================
pull-request: bpf-next 2018-07-20

The following pull-request contains BPF updates for your *net-next* tree.

The main changes are:

1) Add sharing of BPF objects within one ASIC: this allows for reuse of
   the same program on multiple ports of a device, and therefore gains
   better code store utilization. On top of that, this now also enables
   sharing of maps between programs attached to different ports of a
   device, from Jakub.

2) Cleanup in libbpf and bpftool's Makefile to reduce unneeded feature
   detections and unused variable exports, also from Jakub.

3) First batch of RCU annotation fixes in prog array handling, i.e.
   there are several __rcu markers which are not correct as well as
   some of the RCU handling, from Roman.

4) Two fixes in BPF sample files related to checking of the prog_cnt
   upper limit from sample loader, from Dan.

5) Minor cleanup in sockmap to remove a set but not used variable,
   from Colin.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2018-07-20 23:58:30 -07:00
commit eae249b27f
21 changed files with 558 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
drivers/net/ethernet/netronome/nfp/bpf/main.c
View File

@ -404,6 +404,20 @@ static int nfp_bpf_parse_capabilities(struct nfp_app *app)
return -EINVAL;
}
static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
{
struct nfp_app_bpf *bpf = app->priv;
return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
}
static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
{
struct nfp_app_bpf *bpf = app->priv;
bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
}
static int nfp_bpf_init(struct nfp_app *app)
{
struct nfp_app_bpf *bpf;
@ -427,6 +441,11 @@ static int nfp_bpf_init(struct nfp_app *app)
if (err)
goto err_free_neutral_maps;
bpf->bpf_dev = bpf_offload_dev_create();
err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
if (err)
goto err_free_neutral_maps;
return 0;
err_free_neutral_maps:
@ -445,6 +464,7 @@ static void nfp_bpf_clean(struct nfp_app *app)
{
struct nfp_app_bpf *bpf = app->priv;
bpf_offload_dev_destroy(bpf->bpf_dev);
WARN_ON(!skb_queue_empty(&bpf->cmsg_replies));
WARN_ON(!list_empty(&bpf->map_list));
WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
@ -466,6 +486,9 @@ const struct nfp_app_type app_bpf = {
.extra_cap = nfp_bpf_extra_cap,
.ndo_init = nfp_bpf_ndo_init,
.ndo_uninit = nfp_bpf_ndo_uninit,
.vnic_alloc = nfp_bpf_vnic_alloc,
.vnic_free = nfp_bpf_vnic_free,

4
drivers/net/ethernet/netronome/nfp/bpf/main.h
View File

@ -110,6 +110,8 @@ enum pkt_vec {
* struct nfp_app_bpf - bpf app priv structure
* @app: backpointer to the app
*
* @bpf_dev: BPF offload device handle
*
* @tag_allocator: bitmap of control message tags in use
* @tag_alloc_next: next tag bit to allocate
* @tag_alloc_last: next tag bit to be freed
@ -150,6 +152,8 @@ enum pkt_vec {
struct nfp_app_bpf {
struct nfp_app *app;
struct bpf_offload_dev *bpf_dev;
DECLARE_BITMAP(tag_allocator, U16_MAX + 1);
u16 tag_alloc_next;
u16 tag_alloc_last;

10
drivers/net/ethernet/netronome/nfp/bpf/offload.c
View File

@ -566,14 +566,8 @@ int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
{
int err;
if (prog) {
struct bpf_prog_offload *offload = prog->aux->offload;
if (!offload)
return -EINVAL;
if (offload->netdev != nn->dp.netdev)
return -EINVAL;
}
if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev))
return -EINVAL;
if (prog && old_prog) {
u8 cap;

17
drivers/net/ethernet/netronome/nfp/nfp_app.c
View File

@ -86,6 +86,23 @@ const char *nfp_app_mip_name(struct nfp_app *app)
return nfp_mip_name(app->pf->mip);
}
int nfp_app_ndo_init(struct net_device *netdev)
{
struct nfp_app *app = nfp_app_from_netdev(netdev);
if (!app || !app->type->ndo_init)
return 0;
return app->type->ndo_init(app, netdev);
}
void nfp_app_ndo_uninit(struct net_device *netdev)
{
struct nfp_app *app = nfp_app_from_netdev(netdev);
if (app && app->type->ndo_uninit)
app->type->ndo_uninit(app, netdev);
}
u64 *nfp_app_port_get_stats(struct nfp_port *port, u64 *data)
{
if (!port || !port->app || !port->app->type->port_get_stats)

8
drivers/net/ethernet/netronome/nfp/nfp_app.h
View File

@ -78,6 +78,8 @@ extern const struct nfp_app_type app_abm;
* @init: perform basic app checks and init
* @clean: clean app state
* @extra_cap: extra capabilities string
* @ndo_init: vNIC and repr netdev .ndo_init
* @ndo_uninit: vNIC and repr netdev .ndo_unint
* @vnic_alloc: allocate vNICs (assign port types, etc.)
* @vnic_free: free up app's vNIC state
* @vnic_init: vNIC netdev was registered
@ -117,6 +119,9 @@ struct nfp_app_type {
const char *(*extra_cap)(struct nfp_app *app, struct nfp_net *nn);
int (*ndo_init)(struct nfp_app *app, struct net_device *netdev);
void (*ndo_uninit)(struct nfp_app *app, struct net_device *netdev);
int (*vnic_alloc)(struct nfp_app *app, struct nfp_net *nn,
unsigned int id);
void (*vnic_free)(struct nfp_app *app, struct nfp_net *nn);
@ -200,6 +205,9 @@ static inline void nfp_app_clean(struct nfp_app *app)
app->type->clean(app);
}
int nfp_app_ndo_init(struct net_device *netdev);
void nfp_app_ndo_uninit(struct net_device *netdev);
static inline int nfp_app_vnic_alloc(struct nfp_app *app, struct nfp_net *nn,
unsigned int id)
{

2
drivers/net/ethernet/netronome/nfp/nfp_net_common.c
View File

@ -3480,6 +3480,8 @@ static int nfp_net_set_mac_address(struct net_device *netdev, void *addr)
}
const struct net_device_ops nfp_net_netdev_ops = {
.ndo_init = nfp_app_ndo_init,
.ndo_uninit = nfp_app_ndo_uninit,
.ndo_open = nfp_net_netdev_open,
.ndo_stop = nfp_net_netdev_close,
.ndo_start_xmit = nfp_net_tx,

2
drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
View File

@ -262,6 +262,8 @@ static int nfp_repr_open(struct net_device *netdev)
}
const struct net_device_ops nfp_repr_netdev_ops = {
.ndo_init = nfp_app_ndo_init,
.ndo_uninit = nfp_app_ndo_uninit,
.ndo_open = nfp_repr_open,
.ndo_stop = nfp_repr_stop,
.ndo_start_xmit = nfp_repr_xmit,

50
drivers/net/netdevsim/bpf.c
View File

@ -238,8 +238,8 @@ static int nsim_bpf_create_prog(struct netdevsim *ns, struct bpf_prog *prog)
state->state = "verify";
/* Program id is not populated yet when we create the state. */
sprintf(name, "%u", ns->prog_id_gen++);
state->ddir = debugfs_create_dir(name, ns->ddir_bpf_bound_progs);
sprintf(name, "%u", ns->sdev->prog_id_gen++);
state->ddir = debugfs_create_dir(name, ns->sdev->ddir_bpf_bound_progs);
if (IS_ERR_OR_NULL(state->ddir)) {
kfree(state);
return -ENOMEM;
@ -250,7 +250,7 @@ static int nsim_bpf_create_prog(struct netdevsim *ns, struct bpf_prog *prog)
&state->state, &nsim_bpf_string_fops);
debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded);
list_add_tail(&state->l, &ns->bpf_bound_progs);
list_add_tail(&state->l, &ns->sdev->bpf_bound_progs);
prog->aux->offload->dev_priv = state;
@ -294,7 +294,7 @@ nsim_setup_prog_hw_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
NSIM_EA(bpf->extack, "xdpoffload of non-bound program");
return -EINVAL;
}
if (bpf->prog->aux->offload->netdev != ns->netdev) {
if (!bpf_offload_dev_match(bpf->prog, ns->netdev)) {
NSIM_EA(bpf->extack, "program bound to different dev");
return -EINVAL;
}
@ -497,7 +497,7 @@ nsim_bpf_map_alloc(struct netdevsim *ns, struct bpf_offloaded_map *offmap)
}
offmap->dev_ops = &nsim_bpf_map_ops;
list_add_tail(&nmap->l, &ns->bpf_bound_maps);
list_add_tail(&nmap->l, &ns->sdev->bpf_bound_maps);
return 0;
@ -582,8 +582,26 @@ int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
int nsim_bpf_init(struct netdevsim *ns)
{
INIT_LIST_HEAD(&ns->bpf_bound_progs);
INIT_LIST_HEAD(&ns->bpf_bound_maps);
int err;
if (ns->sdev->refcnt == 1) {
INIT_LIST_HEAD(&ns->sdev->bpf_bound_progs);
INIT_LIST_HEAD(&ns->sdev->bpf_bound_maps);
ns->sdev->ddir_bpf_bound_progs =
debugfs_create_dir("bpf_bound_progs", ns->sdev->ddir);
if (IS_ERR_OR_NULL(ns->sdev->ddir_bpf_bound_progs))
return -ENOMEM;
ns->sdev->bpf_dev = bpf_offload_dev_create();
err = PTR_ERR_OR_ZERO(ns->sdev->bpf_dev);
if (err)
return err;
}
err = bpf_offload_dev_netdev_register(ns->sdev->bpf_dev, ns->netdev);
if (err)
goto err_destroy_bdev;
debugfs_create_u32("bpf_offloaded_id", 0400, ns->ddir,
&ns->bpf_offloaded_id);
@ -593,10 +611,6 @@ int nsim_bpf_init(struct netdevsim *ns)
&ns->bpf_bind_accept);
debugfs_create_u32("bpf_bind_verifier_delay", 0600, ns->ddir,
&ns->bpf_bind_verifier_delay);
ns->ddir_bpf_bound_progs =
debugfs_create_dir("bpf_bound_progs", ns->ddir);
if (IS_ERR_OR_NULL(ns->ddir_bpf_bound_progs))
return -ENOMEM;
ns->bpf_tc_accept = true;
debugfs_create_bool("bpf_tc_accept", 0600, ns->ddir,
@ -615,13 +629,23 @@ int nsim_bpf_init(struct netdevsim *ns)
&ns->bpf_map_accept);
return 0;
err_destroy_bdev:
if (ns->sdev->refcnt == 1)
bpf_offload_dev_destroy(ns->sdev->bpf_dev);
return err;
}
void nsim_bpf_uninit(struct netdevsim *ns)
{
WARN_ON(!list_empty(&ns->bpf_bound_progs));
WARN_ON(!list_empty(&ns->bpf_bound_maps));
WARN_ON(ns->xdp.prog);
WARN_ON(ns->xdp_hw.prog);
WARN_ON(ns->bpf_offloaded);
bpf_offload_dev_netdev_unregister(ns->sdev->bpf_dev, ns->netdev);
if (ns->sdev->refcnt == 1) {
WARN_ON(!list_empty(&ns->sdev->bpf_bound_progs));
WARN_ON(!list_empty(&ns->sdev->bpf_bound_maps));
bpf_offload_dev_destroy(ns->sdev->bpf_dev);
}
}

103
drivers/net/netdevsim/netdev.c
View File

@ -22,6 +22,7 @@
#include <net/netlink.h>
#include <net/pkt_cls.h>
#include <net/rtnetlink.h>
#include <net/switchdev.h>
#include "netdevsim.h"
@ -144,8 +145,29 @@ static struct device_type nsim_dev_type = {
.release = nsim_dev_release,
};
static int
nsim_port_attr_get(struct net_device *dev, struct switchdev_attr *attr)
{
struct netdevsim *ns = netdev_priv(dev);
switch (attr->id) {
case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
attr->u.ppid.id_len = sizeof(ns->sdev->switch_id);
memcpy(&attr->u.ppid.id, &ns->sdev->switch_id,
attr->u.ppid.id_len);
return 0;
default:
return -EOPNOTSUPP;
}
}
static const struct switchdev_ops nsim_switchdev_ops = {
.switchdev_port_attr_get = nsim_port_attr_get,
};
static int nsim_init(struct net_device *dev)
{
char sdev_ddir_name[10], sdev_link_name[32];
struct netdevsim *ns = netdev_priv(dev);
int err;
@ -154,9 +176,32 @@ static int nsim_init(struct net_device *dev)
if (IS_ERR_OR_NULL(ns->ddir))
return -ENOMEM;
if (!ns->sdev) {
ns->sdev = kzalloc(sizeof(*ns->sdev), GFP_KERNEL);
if (!ns->sdev) {
err = -ENOMEM;
goto err_debugfs_destroy;
}
ns->sdev->refcnt = 1;
ns->sdev->switch_id = nsim_dev_id;
sprintf(sdev_ddir_name, "%u", ns->sdev->switch_id);
ns->sdev->ddir = debugfs_create_dir(sdev_ddir_name,
nsim_sdev_ddir);
if (IS_ERR_OR_NULL(ns->sdev->ddir)) {
err = PTR_ERR_OR_ZERO(ns->sdev->ddir) ?: -EINVAL;
goto err_sdev_free;
}
} else {
sprintf(sdev_ddir_name, "%u", ns->sdev->switch_id);
ns->sdev->refcnt++;
}
sprintf(sdev_link_name, "../../" DRV_NAME "_sdev/%s", sdev_ddir_name);
debugfs_create_symlink("sdev", ns->ddir, sdev_link_name);
err = nsim_bpf_init(ns);
if (err)
goto err_debugfs_destroy;
goto err_sdev_destroy;
ns->dev.id = nsim_dev_id++;
ns->dev.bus = &nsim_bus;
@ -166,6 +211,7 @@ static int nsim_init(struct net_device *dev)
goto err_bpf_uninit;
SET_NETDEV_DEV(dev, &ns->dev);
SWITCHDEV_SET_OPS(dev, &nsim_switchdev_ops);
err = nsim_devlink_setup(ns);
if (err)
@ -179,6 +225,12 @@ static int nsim_init(struct net_device *dev)
device_unregister(&ns->dev);
err_bpf_uninit:
nsim_bpf_uninit(ns);
err_sdev_destroy:
if (!--ns->sdev->refcnt) {
debugfs_remove_recursive(ns->sdev->ddir);
err_sdev_free:
kfree(ns->sdev);
}
err_debugfs_destroy:
debugfs_remove_recursive(ns->ddir);
return err;
@ -192,6 +244,10 @@ static void nsim_uninit(struct net_device *dev)
nsim_devlink_teardown(ns);
debugfs_remove_recursive(ns->ddir);
nsim_bpf_uninit(ns);
if (!--ns->sdev->refcnt) {
debugfs_remove_recursive(ns->sdev->ddir);
kfree(ns->sdev);
}
}
static void nsim_free(struct net_device *dev)
@ -470,14 +526,48 @@ static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
return 0;
}
static int nsim_newlink(struct net *src_net, struct net_device *dev,
struct nlattr *tb[], struct nlattr *data[],
struct netlink_ext_ack *extack)
{
struct netdevsim *ns = netdev_priv(dev);
if (tb[IFLA_LINK]) {
struct net_device *joindev;
struct netdevsim *joinns;
joindev = __dev_get_by_index(src_net,
nla_get_u32(tb[IFLA_LINK]));
if (!joindev)
return -ENODEV;
if (joindev->netdev_ops != &nsim_netdev_ops)
return -EINVAL;
joinns = netdev_priv(joindev);
if (!joinns->sdev || !joinns->sdev->refcnt)
return -EINVAL;
ns->sdev = joinns->sdev;
}
return register_netdevice(dev);
}
static void nsim_dellink(struct net_device *dev, struct list_head *head)
{
unregister_netdevice_queue(dev, head);
}
static struct rtnl_link_ops nsim_link_ops __read_mostly = {
.kind = DRV_NAME,
.priv_size = sizeof(struct netdevsim),
.setup = nsim_setup,
.validate = nsim_validate,
.newlink = nsim_newlink,
.dellink = nsim_dellink,
};
struct dentry *nsim_ddir;
struct dentry *nsim_sdev_ddir;
static int __init nsim_module_init(void)
{
@ -487,9 +577,15 @@ static int __init nsim_module_init(void)
if (IS_ERR_OR_NULL(nsim_ddir))
return -ENOMEM;
nsim_sdev_ddir = debugfs_create_dir(DRV_NAME "_sdev", NULL);
if (IS_ERR_OR_NULL(nsim_sdev_ddir)) {
err = -ENOMEM;
goto err_debugfs_destroy;
}
err = bus_register(&nsim_bus);
if (err)
goto err_debugfs_destroy;
goto err_sdir_destroy;
err = nsim_devlink_init();
if (err)
@ -505,6 +601,8 @@ static int __init nsim_module_init(void)
nsim_devlink_exit();
err_unreg_bus:
bus_unregister(&nsim_bus);
err_sdir_destroy:
debugfs_remove_recursive(nsim_sdev_ddir);
err_debugfs_destroy:
debugfs_remove_recursive(nsim_ddir);
return err;
@ -515,6 +613,7 @@ static void __exit nsim_module_exit(void)
rtnl_link_unregister(&nsim_link_ops);
nsim_devlink_exit();
bus_unregister(&nsim_bus);
debugfs_remove_recursive(nsim_sdev_ddir);
debugfs_remove_recursive(nsim_ddir);
}

23
drivers/net/netdevsim/netdevsim.h
View File

@ -27,9 +27,25 @@
#define NSIM_EA(extack, msg) NL_SET_ERR_MSG_MOD((extack), msg)
struct bpf_prog;
struct bpf_offload_dev;
struct dentry;
struct nsim_vf_config;
struct netdevsim_shared_dev {
unsigned int refcnt;
u32 switch_id;
struct dentry *ddir;
struct bpf_offload_dev *bpf_dev;
struct dentry *ddir_bpf_bound_progs;
u32 prog_id_gen;
struct list_head bpf_bound_progs;
struct list_head bpf_bound_maps;
};
#define NSIM_IPSEC_MAX_SA_COUNT 33
#define NSIM_IPSEC_VALID BIT(31)
@ -59,6 +75,7 @@ struct netdevsim {
struct u64_stats_sync syncp;
struct device dev;
struct netdevsim_shared_dev *sdev;
struct dentry *ddir;
@ -71,12 +88,8 @@ struct netdevsim {
struct xdp_attachment_info xdp;
struct xdp_attachment_info xdp_hw;
u32 prog_id_gen;
bool bpf_bind_accept;
u32 bpf_bind_verifier_delay;
struct dentry *ddir_bpf_bound_progs;
struct list_head bpf_bound_progs;
bool bpf_tc_accept;
bool bpf_tc_non_bound_accept;
@ -84,7 +97,6 @@ struct netdevsim {
bool bpf_xdpoffload_accept;
bool bpf_map_accept;
struct list_head bpf_bound_maps;
#if IS_ENABLED(CONFIG_NET_DEVLINK)
struct devlink *devlink;
#endif
@ -92,6 +104,7 @@ struct netdevsim {
};
extern struct dentry *nsim_ddir;
extern struct dentry *nsim_sdev_ddir;
#ifdef CONFIG_BPF_SYSCALL
int nsim_bpf_init(struct netdevsim *ns);

13
include/linux/bpf.h
View File

@ -85,6 +85,7 @@ struct bpf_map {
char name[BPF_OBJ_NAME_LEN];
};
struct bpf_offload_dev;
struct bpf_offloaded_map;
struct bpf_map_dev_ops {
@ -352,7 +353,7 @@ struct bpf_prog_array {
struct bpf_prog *progs[0];
};
struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
void bpf_prog_array_free(struct bpf_prog_array __rcu *progs);
int bpf_prog_array_length(struct bpf_prog_array __rcu *progs);
int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
@ -648,7 +649,15 @@ int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
int bpf_map_offload_get_next_key(struct bpf_map *map,
void *key, void *next_key);
bool bpf_offload_dev_match(struct bpf_prog *prog, struct bpf_map *map);
bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
struct bpf_offload_dev *bpf_offload_dev_create(void);
void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
struct net_device *netdev);
void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
struct net_device *netdev);
bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr);

7
kernel/bpf/cgroup.c
View File

@ -95,7 +95,7 @@ static int compute_effective_progs(struct cgroup *cgrp,
enum bpf_attach_type type,
struct bpf_prog_array __rcu **array)
{
struct bpf_prog_array __rcu *progs;
struct bpf_prog_array *progs;
struct bpf_prog_list *pl;
struct cgroup *p = cgrp;
int cnt = 0;
@ -120,13 +120,12 @@ static int compute_effective_progs(struct cgroup *cgrp,
&p->bpf.progs[type], node) {
if (!pl->prog)
continue;
rcu_dereference_protected(progs, 1)->
progs[cnt++] = pl->prog;
progs->progs[cnt++] = pl->prog;
}
p = cgroup_parent(p);
} while (p);
*array = progs;
rcu_assign_pointer(*array, progs);
return 0;
}

2
kernel/bpf/core.c
View File

@ -1538,7 +1538,7 @@ static struct {
.null_prog = NULL,
};
struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
{
if (prog_cnt)
return kzalloc(sizeof(struct bpf_prog_array) +

229
kernel/bpf/offload.c
View File

@ -18,19 +18,43 @@
#include <linux/bug.h>
#include <linux/kdev_t.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/netdevice.h>
#include <linux/printk.h>
#include <linux/proc_ns.h>
#include <linux/rhashtable.h>
#include <linux/rtnetlink.h>
#include <linux/rwsem.h>
/* Protects bpf_prog_offload_devs, bpf_map_offload_devs and offload members
/* Protects offdevs, members of bpf_offload_netdev and offload members
* of all progs.
* RTNL lock cannot be taken when holding this lock.
*/
static DECLARE_RWSEM(bpf_devs_lock);
static LIST_HEAD(bpf_prog_offload_devs);
static LIST_HEAD(bpf_map_offload_devs);
struct bpf_offload_dev {
struct list_head netdevs;
};
struct bpf_offload_netdev {
struct rhash_head l;
struct net_device *netdev;
struct bpf_offload_dev *offdev;
struct list_head progs;
struct list_head maps;
struct list_head offdev_netdevs;
};
static const struct rhashtable_params offdevs_params = {
.nelem_hint = 4,
.key_len = sizeof(struct net_device *),
.key_offset = offsetof(struct bpf_offload_netdev, netdev),
.head_offset = offsetof(struct bpf_offload_netdev, l),
.automatic_shrinking = true,
};
static struct rhashtable offdevs;
static bool offdevs_inited;
static int bpf_dev_offload_check(struct net_device *netdev)
{
@ -41,8 +65,19 @@ static int bpf_dev_offload_check(struct net_device *netdev)
return 0;
}
static struct bpf_offload_netdev *
bpf_offload_find_netdev(struct net_device *netdev)
{
lockdep_assert_held(&bpf_devs_lock);
if (!offdevs_inited)
return NULL;
return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
}
int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr)
{
struct bpf_offload_netdev *ondev;
struct bpf_prog_offload *offload;
int err;
@ -66,12 +101,13 @@ int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr)
goto err_maybe_put;
down_write(&bpf_devs_lock);
if (offload->netdev->reg_state != NETREG_REGISTERED) {
ondev = bpf_offload_find_netdev(offload->netdev);
if (!ondev) {
err = -EINVAL;
goto err_unlock;
}
prog->aux->offload = offload;
list_add_tail(&offload->offloads, &bpf_prog_offload_devs);
list_add_tail(&offload->offloads, &ondev->progs);
dev_put(offload->netdev);
up_write(&bpf_devs_lock);
@ -294,6 +330,7 @@ static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap,
struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
{
struct net *net = current->nsproxy->net_ns;
struct bpf_offload_netdev *ondev;
struct bpf_offloaded_map *offmap;
int err;
@ -316,11 +353,17 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
if (err)
goto err_unlock;
ondev = bpf_offload_find_netdev(offmap->netdev);
if (!ondev) {
err = -EINVAL;
goto err_unlock;
}
err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC);
if (err)
goto err_unlock;
list_add_tail(&offmap->offloads, &bpf_map_offload_devs);
list_add_tail(&offmap->offloads, &ondev->maps);
up_write(&bpf_devs_lock);
rtnl_unlock();
@ -468,77 +511,159 @@ int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map)
return 0;
}
bool bpf_offload_dev_match(struct bpf_prog *prog, struct bpf_map *map)
static bool __bpf_offload_dev_match(struct bpf_prog *prog,
struct net_device *netdev)
{
struct bpf_offloaded_map *offmap;
struct bpf_offload_netdev *ondev1, *ondev2;
struct bpf_prog_offload *offload;
bool ret;
if (!bpf_prog_is_dev_bound(prog->aux))
return false;
if (!bpf_map_is_dev_bound(map))
return bpf_map_offload_neutral(map);
offload = prog->aux->offload;
if (!offload)
return false;
if (offload->netdev == netdev)
return true;
ondev1 = bpf_offload_find_netdev(offload->netdev);
ondev2 = bpf_offload_find_netdev(netdev);
return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev;
}
bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev)
{
bool ret;
down_read(&bpf_devs_lock);
offload = prog->aux->offload;
ret = __bpf_offload_dev_match(prog, netdev);
up_read(&bpf_devs_lock);
return ret;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_match);
bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map)
{
struct bpf_offloaded_map *offmap;
bool ret;
if (!bpf_map_is_dev_bound(map))
return bpf_map_offload_neutral(map);
offmap = map_to_offmap(map);
ret = offload && offload->netdev == offmap->netdev;
down_read(&bpf_devs_lock);
ret = __bpf_offload_dev_match(prog, offmap->netdev);
up_read(&bpf_devs_lock);
return ret;
}
static void bpf_offload_orphan_all_progs(struct net_device *netdev)
int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
struct net_device *netdev)
{
struct bpf_prog_offload *offload, *tmp;
struct bpf_offload_netdev *ondev;
int err;
list_for_each_entry_safe(offload, tmp, &bpf_prog_offload_devs, offloads)
if (offload->netdev == netdev)
__bpf_prog_offload_destroy(offload->prog);
ondev = kzalloc(sizeof(*ondev), GFP_KERNEL);
if (!ondev)
return -ENOMEM;
ondev->netdev = netdev;
ondev->offdev = offdev;
INIT_LIST_HEAD(&ondev->progs);
INIT_LIST_HEAD(&ondev->maps);
down_write(&bpf_devs_lock);
err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params);
if (err) {
netdev_warn(netdev, "failed to register for BPF offload\n");
goto err_unlock_free;
}
list_add(&ondev->offdev_netdevs, &offdev->netdevs);
up_write(&bpf_devs_lock);
return 0;
err_unlock_free:
up_write(&bpf_devs_lock);
kfree(ondev);
return err;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register);
static void bpf_offload_orphan_all_maps(struct net_device *netdev)
void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
struct net_device *netdev)
{
struct bpf_offloaded_map *offmap, *tmp;
list_for_each_entry_safe(offmap, tmp, &bpf_map_offload_devs, offloads)
if (offmap->netdev == netdev)
__bpf_map_offload_destroy(offmap);
}
static int bpf_offload_notification(struct notifier_block *notifier,
ulong event, void *ptr)
{
struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
struct bpf_offload_netdev *ondev, *altdev;
struct bpf_offloaded_map *offmap, *mtmp;
struct bpf_prog_offload *offload, *ptmp;
ASSERT_RTNL();
switch (event) {
case NETDEV_UNREGISTER:
/* ignore namespace changes */
if (netdev->reg_state != NETREG_UNREGISTERING)
break;
down_write(&bpf_devs_lock);
ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
if (WARN_ON(!ondev))
goto unlock;
down_write(&bpf_devs_lock);
bpf_offload_orphan_all_progs(netdev);
bpf_offload_orphan_all_maps(netdev);
up_write(&bpf_devs_lock);
break;
default:
break;
WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params));
list_del(&ondev->offdev_netdevs);
/* Try to move the objects to another netdev of the device */
altdev = list_first_entry_or_null(&offdev->netdevs,
struct bpf_offload_netdev,
offdev_netdevs);
if (altdev) {
list_for_each_entry(offload, &ondev->progs, offloads)
offload->netdev = altdev->netdev;
list_splice_init(&ondev->progs, &altdev->progs);
list_for_each_entry(offmap, &ondev->maps, offloads)
offmap->netdev = altdev->netdev;
list_splice_init(&ondev->maps, &altdev->maps);
} else {
list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads)
__bpf_prog_offload_destroy(offload->prog);
list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads)
__bpf_map_offload_destroy(offmap);
}
return NOTIFY_OK;
WARN_ON(!list_empty(&ondev->progs));
WARN_ON(!list_empty(&ondev->maps));
kfree(ondev);
unlock:
up_write(&bpf_devs_lock);
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister);
static struct notifier_block bpf_offload_notifier = {
.notifier_call = bpf_offload_notification,
};
static int __init bpf_offload_init(void)
struct bpf_offload_dev *bpf_offload_dev_create(void)
{
register_netdevice_notifier(&bpf_offload_notifier);
return 0;
}
struct bpf_offload_dev *offdev;
int err;
subsys_initcall(bpf_offload_init);
down_write(&bpf_devs_lock);
if (!offdevs_inited) {
err = rhashtable_init(&offdevs, &offdevs_params);
if (err)
return ERR_PTR(err);
offdevs_inited = true;
}
up_write(&bpf_devs_lock);
offdev = kzalloc(sizeof(*offdev), GFP_KERNEL);
if (!offdev)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&offdev->netdevs);
return offdev;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_create);
void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev)
{
WARN_ON(!list_empty(&offdev->netdevs));
kfree(offdev);
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy);

3
kernel/bpf/sockmap.c
View File

@ -725,11 +725,8 @@ static int bpf_tcp_sendmsg_do_redirect(struct sock *sk, int send,
{
bool ingress = !!(md->flags & BPF_F_INGRESS);
struct smap_psock *psock;
struct scatterlist *sg;
int err = 0;
sg = md->sg_data;
rcu_read_lock();
psock = smap_psock_sk(sk);
if (unlikely(!psock))

2
kernel/bpf/verifier.c
View File

@ -5054,7 +5054,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,
}
if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
!bpf_offload_dev_match(prog, map)) {
!bpf_offload_prog_map_match(prog, map)) {
verbose(env, "offload device mismatch between prog and map\n");
return -EINVAL;
}

3
samples/bpf/bpf_load.c
View File

@ -107,6 +107,9 @@ static int load_and_attach(const char *event, struct bpf_insn *prog, int size)
return -1;
}
if (prog_cnt == MAX_PROGS)
return -1;
fd = bpf_load_program(prog_type, prog, insns_cnt, license, kern_version,
bpf_log_buf, BPF_LOG_BUF_SIZE);
if (fd < 0) {

2
samples/bpf/test_cgrp2_sock2.c
View File

@ -51,7 +51,7 @@ int main(int argc, char **argv)
if (argc > 3)
filter_id = atoi(argv[3]);
if (filter_id > prog_cnt) {
if (filter_id >= prog_cnt) {
printf("Invalid program id; program not found in file\n");
return EXIT_FAILURE;
}

2
tools/bpf/bpftool/Makefile
View File

@ -26,7 +26,7 @@ LIBBPF = $(BPF_PATH)libbpf.a
BPFTOOL_VERSION := $(shell make --no-print-directory -sC ../../.. kernelversion)
$(LIBBPF): FORCE
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(OUTPUT) $(OUTPUT)libbpf.a FEATURES_DUMP=$(FEATURE_DUMP_EXPORT)
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(OUTPUT) $(OUTPUT)libbpf.a
$(LIBBPF)-clean:
$(call QUIET_CLEAN, libbpf)

6
tools/lib/bpf/Makefile
View File

@ -66,7 +66,7 @@ ifndef VERBOSE
endif
FEATURE_USER = .libbpf
FEATURE_TESTS = libelf libelf-getphdrnum libelf-mmap bpf reallocarray
FEATURE_TESTS = libelf libelf-mmap bpf reallocarray
FEATURE_DISPLAY = libelf bpf
INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/arch/$(ARCH)/include/uapi -I$(srctree)/tools/include/uapi -I$(srctree)/tools/perf
@ -116,10 +116,6 @@ ifeq ($(feature-libelf-mmap), 1)
override CFLAGS += -DHAVE_LIBELF_MMAP_SUPPORT
endif
ifeq ($(feature-libelf-getphdrnum), 1)
override CFLAGS += -DHAVE_ELF_GETPHDRNUM_SUPPORT
endif
ifeq ($(feature-reallocarray), 0)
override CFLAGS += -DCOMPAT_NEED_REALLOCARRAY
endif

151
tools/testing/selftests/bpf/test_offload.py
View File

@ -158,8 +158,9 @@ def tool(name, args, flags, JSON=True, ns="", fail=True, include_stderr=False):
else:
return ret, out
def bpftool(args, JSON=True, ns="", fail=True):
return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns, fail=fail)
def bpftool(args, JSON=True, ns="", fail=True, include_stderr=False):
return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns,
fail=fail, include_stderr=include_stderr)
def bpftool_prog_list(expected=None, ns=""):
_, progs = bpftool("prog show", JSON=True, ns=ns, fail=True)
@ -201,6 +202,21 @@ def bpftool_map_list_wait(expected=0, n_retry=20):
time.sleep(0.05)
raise Exception("Time out waiting for map counts to stabilize want %d, have %d" % (expected, nmaps))
def bpftool_prog_load(sample, file_name, maps=[], prog_type="xdp", dev=None,
fail=True, include_stderr=False):
args = "prog load %s %s" % (os.path.join(bpf_test_dir, sample), file_name)
if prog_type is not None:
args += " type " + prog_type
if dev is not None:
args += " dev " + dev
if len(maps):
args += " map " + " map ".join(maps)
res = bpftool(args, fail=fail, include_stderr=include_stderr)
if res[0] == 0:
files.append(file_name)
return res
def ip(args, force=False, JSON=True, ns="", fail=True, include_stderr=False):
if force:
args = "-force " + args
@ -307,21 +323,25 @@ class NetdevSim:
Class for netdevsim netdevice and its attributes.
"""
def __init__(self):
def __init__(self, link=None):
self.link = link
self.dev = self._netdevsim_create()
devs.append(self)
self.ns = ""
self.dfs_dir = '/sys/kernel/debug/netdevsim/%s' % (self.dev['ifname'])
self.sdev_dir = self.dfs_dir + '/sdev/'
self.dfs_refresh()
def __getitem__(self, key):
return self.dev[key]
def _netdevsim_create(self):
link = "" if self.link is None else "link " + self.link.dev['ifname']
_, old = ip("link show")
ip("link add sim%d type netdevsim")
ip("link add sim%d {link} type netdevsim".format(link=link))
_, new = ip("link show")
for dev in new:
@ -345,12 +365,12 @@ class NetdevSim:
return data.strip()
def dfs_num_bound_progs(self):
path = os.path.join(self.dfs_dir, "bpf_bound_progs")
path = os.path.join(self.sdev_dir, "bpf_bound_progs")
_, progs = cmd('ls %s' % (path))
return len(progs.split())
def dfs_get_bound_progs(self, expected):
progs = DebugfsDir(os.path.join(self.dfs_dir, "bpf_bound_progs"))
progs = DebugfsDir(os.path.join(self.sdev_dir, "bpf_bound_progs"))
if expected is not None:
if len(progs) != expected:
fail(True, "%d BPF programs bound, expected %d" %
@ -847,6 +867,25 @@ try:
sim.set_mtu(1500)
sim.wait_for_flush()
start_test("Test non-offload XDP attaching to HW...")
bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/nooffload")
nooffload = bpf_pinned("/sys/fs/bpf/nooffload")
ret, _, err = sim.set_xdp(nooffload, "offload",
fail=False, include_stderr=True)
fail(ret == 0, "attached non-offloaded XDP program to HW")
check_extack_nsim(err, "xdpoffload of non-bound program.", args)
rm("/sys/fs/bpf/nooffload")
start_test("Test offload XDP attaching to drv...")
bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",
dev=sim['ifname'])
offload = bpf_pinned("/sys/fs/bpf/offload")
ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)
fail(ret == 0, "attached offloaded XDP program to drv")
check_extack(err, "using device-bound program without HW_MODE flag is not supported.", args)
rm("/sys/fs/bpf/offload")
sim.wait_for_flush()
start_test("Test XDP offload...")
_, _, err = sim.set_xdp(obj, "offload", verbose=True, include_stderr=True)
ipl = sim.ip_link_show(xdp=True)
@ -1140,6 +1179,106 @@ try:
fail(ret == 0,
"netdevsim didn't refuse to create a map with offload disabled")
sim.remove()
start_test("Test multi-dev ASIC program reuse...")
simA = NetdevSim()
simB1 = NetdevSim()
simB2 = NetdevSim(link=simB1)
simB3 = NetdevSim(link=simB1)
sims = (simA, simB1, simB2, simB3)
simB = (simB1, simB2, simB3)
bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA",
dev=simA['ifname'])
progA = bpf_pinned("/sys/fs/bpf/nsimA")
bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB",
dev=simB1['ifname'])
progB = bpf_pinned("/sys/fs/bpf/nsimB")
simA.set_xdp(progA, "offload", JSON=False)
for d in simB:
d.set_xdp(progB, "offload", JSON=False)
start_test("Test multi-dev ASIC cross-dev replace...")
ret, _ = simA.set_xdp(progB, "offload", force=True, JSON=False, fail=False)
fail(ret == 0, "cross-ASIC program allowed")
for d in simB:
ret, _ = d.set_xdp(progA, "offload", force=True, JSON=False, fail=False)
fail(ret == 0, "cross-ASIC program allowed")
start_test("Test multi-dev ASIC cross-dev install...")
for d in sims:
d.unset_xdp("offload")
ret, _, err = simA.set_xdp(progB, "offload", force=True, JSON=False,
fail=False, include_stderr=True)
fail(ret == 0, "cross-ASIC program allowed")
check_extack_nsim(err, "program bound to different dev.", args)
for d in simB:
ret, _, err = d.set_xdp(progA, "offload", force=True, JSON=False,
fail=False, include_stderr=True)
fail(ret == 0, "cross-ASIC program allowed")
check_extack_nsim(err, "program bound to different dev.", args)
start_test("Test multi-dev ASIC cross-dev map reuse...")
mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]
mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]
ret, _ = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
dev=simB3['ifname'],
maps=["idx 0 id %d" % (mapB)],
fail=False)
fail(ret != 0, "couldn't reuse a map on the same ASIC")
rm("/sys/fs/bpf/nsimB_")
ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA_",
dev=simA['ifname'],
maps=["idx 0 id %d" % (mapB)],
fail=False, include_stderr=True)
fail(ret == 0, "could reuse a map on a different ASIC")
fail(err.count("offload device mismatch between prog and map") == 0,
"error message missing for cross-ASIC map")
ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
dev=simB1['ifname'],
maps=["idx 0 id %d" % (mapA)],
fail=False, include_stderr=True)
fail(ret == 0, "could reuse a map on a different ASIC")
fail(err.count("offload device mismatch between prog and map") == 0,
"error message missing for cross-ASIC map")
start_test("Test multi-dev ASIC cross-dev destruction...")
bpftool_prog_list_wait(expected=2)
simA.remove()
bpftool_prog_list_wait(expected=1)
ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
fail(ifnameB != simB1['ifname'], "program not bound to originial device")
simB1.remove()
bpftool_prog_list_wait(expected=1)
start_test("Test multi-dev ASIC cross-dev destruction - move...")
ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
fail(ifnameB not in (simB2['ifname'], simB3['ifname']),
"program not bound to remaining devices")
simB2.remove()
ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
fail(ifnameB != simB3['ifname'], "program not bound to remaining device")
simB3.remove()
bpftool_prog_list_wait(expected=0)
start_test("Test multi-dev ASIC cross-dev destruction - orphaned...")
ret, out = bpftool("prog show %s" % (progB), fail=False)
fail(ret == 0, "got information about orphaned program")
fail("error" not in out, "no error reported for get info on orphaned")
fail(out["error"] != "can't get prog info: No such device",
"wrong error for get info on orphaned")
print("%s: OK" % (os.path.basename(__file__)))
finally: