2 * Copyright (C) 2017-2018 Netronome Systems, Inc.
4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree or the BSD 2-Clause License provided below. You have the
7 * option to license this software under the complete terms of either license.
9 * The BSD 2-Clause License:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * 1. Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * 2. Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <net/pkt_cls.h>
36 #include "../nfpcore/nfp_cpp.h"
37 #include "../nfpcore/nfp_nffw.h"
38 #include "../nfpcore/nfp_nsp.h"
39 #include "../nfp_app.h"
40 #include "../nfp_main.h"
41 #include "../nfp_net.h"
42 #include "../nfp_port.h"
46 const struct rhashtable_params nfp_bpf_maps_neutral_params = {
48 .key_len = FIELD_SIZEOF(struct nfp_bpf_neutral_map, ptr),
49 .key_offset = offsetof(struct nfp_bpf_neutral_map, ptr),
50 .head_offset = offsetof(struct nfp_bpf_neutral_map, l),
51 .automatic_shrinking = true,
54 static bool nfp_net_ebpf_capable(struct nfp_net *nn)
56 #ifdef __LITTLE_ENDIAN
57 if (nn->cap & NFP_NET_CFG_CTRL_BPF &&
58 nn_readb(nn, NFP_NET_CFG_BPF_ABI) == NFP_NET_BPF_ABI)
65 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
66 struct bpf_prog *prog, struct netlink_ext_ack *extack)
68 bool running, xdp_running;
71 if (!nfp_net_ebpf_capable(nn))
74 running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
75 xdp_running = running && nn->dp.bpf_offload_xdp;
77 if (!prog && !xdp_running)
79 if (prog && running && !xdp_running)
82 ret = nfp_net_bpf_offload(nn, prog, running, extack);
83 /* Stop offload if replace not possible */
85 nfp_bpf_xdp_offload(app, nn, NULL, extack);
87 nn->dp.bpf_offload_xdp = prog && !ret;
91 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
93 return nfp_net_ebpf_capable(nn) ? "BPF" : "";
97 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
99 struct nfp_pf *pf = app->pf;
100 struct nfp_bpf_vnic *bv;
104 nfp_err(pf->cpp, "No ETH table\n");
107 if (pf->max_data_vnics != pf->eth_tbl->count) {
108 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
109 pf->max_data_vnics, pf->eth_tbl->count);
113 bv = kzalloc(sizeof(*bv), GFP_KERNEL);
118 err = nfp_app_nic_vnic_alloc(app, nn, id);
122 bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
123 bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
131 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
133 struct nfp_bpf_vnic *bv = nn->app_priv;
135 WARN_ON(bv->tc_prog);
139 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
140 void *type_data, void *cb_priv)
142 struct tc_cls_bpf_offload *cls_bpf = type_data;
143 struct nfp_net *nn = cb_priv;
144 struct bpf_prog *oldprog;
145 struct nfp_bpf_vnic *bv;
148 if (type != TC_SETUP_CLSBPF) {
149 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
150 "only offload of BPF classifiers supported");
153 if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
155 if (!nfp_net_ebpf_capable(nn)) {
156 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
157 "NFP firmware does not support eBPF offload");
160 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
161 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
162 "only ETH_P_ALL supported as filter protocol");
166 /* Only support TC direct action */
167 if (!cls_bpf->exts_integrated ||
168 tcf_exts_has_actions(cls_bpf->exts)) {
169 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
170 "only direct action with no legacy actions supported");
174 if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
178 oldprog = cls_bpf->oldprog;
180 /* Don't remove if oldprog doesn't match driver's state */
181 if (bv->tc_prog != oldprog) {
187 err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
188 cls_bpf->common.extack);
192 bv->tc_prog = cls_bpf->prog;
193 nn->port->tc_offload_cnt = !!bv->tc_prog;
197 static int nfp_bpf_setup_tc_block(struct net_device *netdev,
198 struct tc_block_offload *f)
200 struct nfp_net *nn = netdev_priv(netdev);
202 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
205 switch (f->command) {
207 return tcf_block_cb_register(f->block,
208 nfp_bpf_setup_tc_block_cb,
210 case TC_BLOCK_UNBIND:
211 tcf_block_cb_unregister(f->block,
212 nfp_bpf_setup_tc_block_cb,
220 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
221 enum tc_setup_type type, void *type_data)
225 return nfp_bpf_setup_tc_block(netdev, type_data);
232 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
234 struct nfp_net *nn = netdev_priv(netdev);
235 unsigned int max_mtu;
237 if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
240 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
241 if (new_mtu > max_mtu) {
242 nn_info(nn, "BPF offload active, MTU over %u not supported\n",
250 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
253 struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
254 struct nfp_cpp *cpp = bpf->app->pf->cpp;
256 if (length < sizeof(*cap)) {
257 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
261 bpf->adjust_head.flags = readl(&cap->flags);
262 bpf->adjust_head.off_min = readl(&cap->off_min);
263 bpf->adjust_head.off_max = readl(&cap->off_max);
264 bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
265 bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
267 if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
268 nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
271 if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
272 !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
273 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
274 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
282 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
284 struct nfp_bpf_cap_tlv_func __iomem *cap = value;
286 if (length < sizeof(*cap)) {
287 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
291 switch (readl(&cap->func_id)) {
292 case BPF_FUNC_map_lookup_elem:
293 bpf->helpers.map_lookup = readl(&cap->func_addr);
295 case BPF_FUNC_map_update_elem:
296 bpf->helpers.map_update = readl(&cap->func_addr);
298 case BPF_FUNC_map_delete_elem:
299 bpf->helpers.map_delete = readl(&cap->func_addr);
301 case BPF_FUNC_perf_event_output:
302 bpf->helpers.perf_event_output = readl(&cap->func_addr);
310 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
312 struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
314 if (length < sizeof(*cap)) {
315 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
319 bpf->maps.types = readl(&cap->types);
320 bpf->maps.max_maps = readl(&cap->max_maps);
321 bpf->maps.max_elems = readl(&cap->max_elems);
322 bpf->maps.max_key_sz = readl(&cap->max_key_sz);
323 bpf->maps.max_val_sz = readl(&cap->max_val_sz);
324 bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
330 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
333 bpf->pseudo_random = true;
338 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
340 bpf->queue_select = true;
344 static int nfp_bpf_parse_capabilities(struct nfp_app *app)
346 struct nfp_cpp *cpp = app->pf->cpp;
347 struct nfp_cpp_area *area;
348 u8 __iomem *mem, *start;
350 mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
353 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
356 while (mem - start + 8 <= nfp_cpp_area_size(area)) {
361 length = readl(mem + 4);
365 if (mem - start > nfp_cpp_area_size(area))
366 goto err_release_free;
369 case NFP_BPF_CAP_TYPE_FUNC:
370 if (nfp_bpf_parse_cap_func(app->priv, value, length))
371 goto err_release_free;
373 case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
374 if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
376 goto err_release_free;
378 case NFP_BPF_CAP_TYPE_MAPS:
379 if (nfp_bpf_parse_cap_maps(app->priv, value, length))
380 goto err_release_free;
382 case NFP_BPF_CAP_TYPE_RANDOM:
383 if (nfp_bpf_parse_cap_random(app->priv, value, length))
384 goto err_release_free;
386 case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
387 if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
388 goto err_release_free;
391 nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
395 if (mem - start != nfp_cpp_area_size(area)) {
396 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
397 mem - start, nfp_cpp_area_size(area));
398 goto err_release_free;
401 nfp_cpp_area_release_free(area);
406 nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
407 nfp_cpp_area_release_free(area);
411 static int nfp_bpf_init(struct nfp_app *app)
413 struct nfp_app_bpf *bpf;
416 bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
422 skb_queue_head_init(&bpf->cmsg_replies);
423 init_waitqueue_head(&bpf->cmsg_wq);
424 INIT_LIST_HEAD(&bpf->map_list);
426 err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
430 err = nfp_bpf_parse_capabilities(app);
432 goto err_free_neutral_maps;
436 err_free_neutral_maps:
437 rhashtable_destroy(&bpf->maps_neutral);
443 static void nfp_check_rhashtable_empty(void *ptr, void *arg)
448 static void nfp_bpf_clean(struct nfp_app *app)
450 struct nfp_app_bpf *bpf = app->priv;
452 WARN_ON(!skb_queue_empty(&bpf->cmsg_replies));
453 WARN_ON(!list_empty(&bpf->map_list));
454 WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
455 rhashtable_free_and_destroy(&bpf->maps_neutral,
456 nfp_check_rhashtable_empty, NULL);
460 const struct nfp_app_type app_bpf = {
461 .id = NFP_APP_BPF_NIC,
466 .init = nfp_bpf_init,
467 .clean = nfp_bpf_clean,
469 .check_mtu = nfp_bpf_check_mtu,
471 .extra_cap = nfp_bpf_extra_cap,
473 .vnic_alloc = nfp_bpf_vnic_alloc,
474 .vnic_free = nfp_bpf_vnic_free,
476 .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx,
478 .setup_tc = nfp_bpf_setup_tc,
480 .xdp_offload = nfp_bpf_xdp_offload,